| Line Number |
../DebugInfoTest/example_mips_dbg.ll
|
Hit count |
Line Number |
../DebugInfoTest/example_mips.ll
|
Hit count |
| 1 |
//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===// |
--- |
1 |
//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===// |
--- |
| 2 |
// |
--- |
2 |
// |
--- |
| 3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
--- |
3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
--- |
| 4 |
// See https://llvm.org/LICENSE.txt for license information. |
--- |
4 |
// See https://llvm.org/LICENSE.txt for license information. |
--- |
| 5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
--- |
5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
--- |
| 6 |
// |
--- |
6 |
// |
--- |
| 7 |
//===----------------------------------------------------------------------===// |
--- |
7 |
//===----------------------------------------------------------------------===// |
--- |
| 8 |
// |
--- |
8 |
// |
--- |
| 9 |
// This implements the SelectionDAG class. |
--- |
9 |
// This implements the SelectionDAG class. |
--- |
| 10 |
// |
--- |
10 |
// |
--- |
| 11 |
//===----------------------------------------------------------------------===// |
--- |
11 |
//===----------------------------------------------------------------------===// |
--- |
| 12 |
|
--- |
12 |
|
--- |
| 13 |
#include "llvm/CodeGen/SelectionDAG.h" |
--- |
13 |
#include "llvm/CodeGen/SelectionDAG.h" |
--- |
| 14 |
#include "SDNodeDbgValue.h" |
--- |
14 |
#include "SDNodeDbgValue.h" |
--- |
| 15 |
#include "llvm/ADT/APFloat.h" |
--- |
15 |
#include "llvm/ADT/APFloat.h" |
--- |
| 16 |
#include "llvm/ADT/APInt.h" |
--- |
16 |
#include "llvm/ADT/APInt.h" |
--- |
| 17 |
#include "llvm/ADT/APSInt.h" |
--- |
17 |
#include "llvm/ADT/APSInt.h" |
--- |
| 18 |
#include "llvm/ADT/ArrayRef.h" |
--- |
18 |
#include "llvm/ADT/ArrayRef.h" |
--- |
| 19 |
#include "llvm/ADT/BitVector.h" |
--- |
19 |
#include "llvm/ADT/BitVector.h" |
--- |
| 20 |
#include "llvm/ADT/DenseSet.h" |
--- |
20 |
#include "llvm/ADT/DenseSet.h" |
--- |
| 21 |
#include "llvm/ADT/FoldingSet.h" |
--- |
21 |
#include "llvm/ADT/FoldingSet.h" |
--- |
| 22 |
#include "llvm/ADT/STLExtras.h" |
--- |
22 |
#include "llvm/ADT/STLExtras.h" |
--- |
| 23 |
#include "llvm/ADT/SmallPtrSet.h" |
--- |
23 |
#include "llvm/ADT/SmallPtrSet.h" |
--- |
| 24 |
#include "llvm/ADT/SmallVector.h" |
--- |
24 |
#include "llvm/ADT/SmallVector.h" |
--- |
| 25 |
#include "llvm/ADT/Twine.h" |
--- |
25 |
#include "llvm/ADT/Twine.h" |
--- |
| 26 |
#include "llvm/Analysis/AliasAnalysis.h" |
--- |
26 |
#include "llvm/Analysis/AliasAnalysis.h" |
--- |
| 27 |
#include "llvm/Analysis/MemoryLocation.h" |
--- |
27 |
#include "llvm/Analysis/MemoryLocation.h" |
--- |
| 28 |
#include "llvm/Analysis/ValueTracking.h" |
--- |
28 |
#include "llvm/Analysis/ValueTracking.h" |
--- |
| 29 |
#include "llvm/Analysis/VectorUtils.h" |
--- |
29 |
#include "llvm/Analysis/VectorUtils.h" |
--- |
| 30 |
#include "llvm/CodeGen/Analysis.h" |
--- |
30 |
#include "llvm/CodeGen/Analysis.h" |
--- |
| 31 |
#include "llvm/CodeGen/FunctionLoweringInfo.h" |
--- |
31 |
#include "llvm/CodeGen/FunctionLoweringInfo.h" |
--- |
| 32 |
#include "llvm/CodeGen/ISDOpcodes.h" |
--- |
32 |
#include "llvm/CodeGen/ISDOpcodes.h" |
--- |
| 33 |
#include "llvm/CodeGen/MachineBasicBlock.h" |
--- |
33 |
#include "llvm/CodeGen/MachineBasicBlock.h" |
--- |
| 34 |
#include "llvm/CodeGen/MachineConstantPool.h" |
--- |
34 |
#include "llvm/CodeGen/MachineConstantPool.h" |
--- |
| 35 |
#include "llvm/CodeGen/MachineFrameInfo.h" |
--- |
35 |
#include "llvm/CodeGen/MachineFrameInfo.h" |
--- |
| 36 |
#include "llvm/CodeGen/MachineFunction.h" |
--- |
36 |
#include "llvm/CodeGen/MachineFunction.h" |
--- |
| 37 |
#include "llvm/CodeGen/MachineMemOperand.h" |
--- |
37 |
#include "llvm/CodeGen/MachineMemOperand.h" |
--- |
| 38 |
#include "llvm/CodeGen/MachineValueType.h" |
--- |
38 |
#include "llvm/CodeGen/MachineValueType.h" |
--- |
| 39 |
#include "llvm/CodeGen/RuntimeLibcalls.h" |
--- |
39 |
#include "llvm/CodeGen/RuntimeLibcalls.h" |
--- |
| 40 |
#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h" |
--- |
40 |
#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h" |
--- |
| 41 |
#include "llvm/CodeGen/SelectionDAGNodes.h" |
--- |
41 |
#include "llvm/CodeGen/SelectionDAGNodes.h" |
--- |
| 42 |
#include "llvm/CodeGen/SelectionDAGTargetInfo.h" |
--- |
42 |
#include "llvm/CodeGen/SelectionDAGTargetInfo.h" |
--- |
| 43 |
#include "llvm/CodeGen/TargetFrameLowering.h" |
--- |
43 |
#include "llvm/CodeGen/TargetFrameLowering.h" |
--- |
| 44 |
#include "llvm/CodeGen/TargetLowering.h" |
--- |
44 |
#include "llvm/CodeGen/TargetLowering.h" |
--- |
| 45 |
#include "llvm/CodeGen/TargetRegisterInfo.h" |
--- |
45 |
#include "llvm/CodeGen/TargetRegisterInfo.h" |
--- |
| 46 |
#include "llvm/CodeGen/TargetSubtargetInfo.h" |
--- |
46 |
#include "llvm/CodeGen/TargetSubtargetInfo.h" |
--- |
| 47 |
#include "llvm/CodeGen/ValueTypes.h" |
--- |
47 |
#include "llvm/CodeGen/ValueTypes.h" |
--- |
| 48 |
#include "llvm/IR/Constant.h" |
--- |
48 |
#include "llvm/IR/Constant.h" |
--- |
| 49 |
#include "llvm/IR/ConstantRange.h" |
--- |
49 |
#include "llvm/IR/ConstantRange.h" |
--- |
| 50 |
#include "llvm/IR/Constants.h" |
--- |
50 |
#include "llvm/IR/Constants.h" |
--- |
| 51 |
#include "llvm/IR/DataLayout.h" |
--- |
51 |
#include "llvm/IR/DataLayout.h" |
--- |
| 52 |
#include "llvm/IR/DebugInfoMetadata.h" |
--- |
52 |
#include "llvm/IR/DebugInfoMetadata.h" |
--- |
| 53 |
#include "llvm/IR/DebugLoc.h" |
--- |
53 |
#include "llvm/IR/DebugLoc.h" |
--- |
| 54 |
#include "llvm/IR/DerivedTypes.h" |
--- |
54 |
#include "llvm/IR/DerivedTypes.h" |
--- |
| 55 |
#include "llvm/IR/Function.h" |
--- |
55 |
#include "llvm/IR/Function.h" |
--- |
| 56 |
#include "llvm/IR/GlobalValue.h" |
--- |
56 |
#include "llvm/IR/GlobalValue.h" |
--- |
| 57 |
#include "llvm/IR/Metadata.h" |
--- |
57 |
#include "llvm/IR/Metadata.h" |
--- |
| 58 |
#include "llvm/IR/Type.h" |
--- |
58 |
#include "llvm/IR/Type.h" |
--- |
| 59 |
#include "llvm/Support/Casting.h" |
--- |
59 |
#include "llvm/Support/Casting.h" |
--- |
| 60 |
#include "llvm/Support/CodeGen.h" |
--- |
60 |
#include "llvm/Support/CodeGen.h" |
--- |
| 61 |
#include "llvm/Support/Compiler.h" |
--- |
61 |
#include "llvm/Support/Compiler.h" |
--- |
| 62 |
#include "llvm/Support/Debug.h" |
--- |
62 |
#include "llvm/Support/Debug.h" |
--- |
| 63 |
#include "llvm/Support/ErrorHandling.h" |
--- |
63 |
#include "llvm/Support/ErrorHandling.h" |
--- |
| 64 |
#include "llvm/Support/KnownBits.h" |
--- |
64 |
#include "llvm/Support/KnownBits.h" |
--- |
| 65 |
#include "llvm/Support/MathExtras.h" |
--- |
65 |
#include "llvm/Support/MathExtras.h" |
--- |
| 66 |
#include "llvm/Support/Mutex.h" |
--- |
66 |
#include "llvm/Support/Mutex.h" |
--- |
| 67 |
#include "llvm/Support/raw_ostream.h" |
--- |
67 |
#include "llvm/Support/raw_ostream.h" |
--- |
| 68 |
#include "llvm/Target/TargetMachine.h" |
--- |
68 |
#include "llvm/Target/TargetMachine.h" |
--- |
| 69 |
#include "llvm/Target/TargetOptions.h" |
--- |
69 |
#include "llvm/Target/TargetOptions.h" |
--- |
| 70 |
#include "llvm/TargetParser/Triple.h" |
--- |
70 |
#include "llvm/TargetParser/Triple.h" |
--- |
| 71 |
#include "llvm/Transforms/Utils/SizeOpts.h" |
--- |
71 |
#include "llvm/Transforms/Utils/SizeOpts.h" |
--- |
| 72 |
#include |
--- |
72 |
#include |
--- |
| 73 |
#include |
--- |
73 |
#include |
--- |
| 74 |
#include |
--- |
74 |
#include |
--- |
| 75 |
#include |
--- |
75 |
#include |
--- |
| 76 |
#include |
--- |
76 |
#include |
--- |
| 77 |
#include |
--- |
77 |
#include |
--- |
| 78 |
#include |
--- |
78 |
#include |
--- |
| 79 |
#include |
--- |
79 |
#include |
--- |
| 80 |
#include |
--- |
80 |
#include |
--- |
| 81 |
|
--- |
81 |
|
--- |
| 82 |
using namespace llvm; |
--- |
82 |
using namespace llvm; |
--- |
| 83 |
|
--- |
83 |
|
--- |
| 84 |
/// makeVTList - Return an instance of the SDVTList struct initialized with the |
--- |
84 |
/// makeVTList - Return an instance of the SDVTList struct initialized with the |
--- |
| 85 |
/// specified members. |
--- |
85 |
/// specified members. |
--- |
| 86 |
static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { |
87 |
86 |
static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { |
87 |
| 87 |
SDVTList Res = {VTs, NumVTs}; |
87 |
87 |
SDVTList Res = {VTs, NumVTs}; |
87 |
| 88 |
return Res; |
87 |
88 |
return Res; |
87 |
| 89 |
} |
--- |
89 |
} |
--- |
| 90 |
|
--- |
90 |
|
--- |
| 91 |
// Default null implementations of the callbacks. |
--- |
91 |
// Default null implementations of the callbacks. |
--- |
| 92 |
void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {} |
0 |
92 |
void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {} |
0 |
| 93 |
void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {} |
7 |
93 |
void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {} |
7 |
| 94 |
void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {} |
3 |
94 |
void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {} |
3 |
| 95 |
|
--- |
95 |
|
--- |
| 96 |
void SelectionDAG::DAGNodeDeletedListener::anchor() {} |
0 |
96 |
void SelectionDAG::DAGNodeDeletedListener::anchor() {} |
0 |
| 97 |
void SelectionDAG::DAGNodeInsertedListener::anchor() {} |
0 |
97 |
void SelectionDAG::DAGNodeInsertedListener::anchor() {} |
0 |
| 98 |
|
--- |
98 |
|
--- |
| 99 |
#define DEBUG_TYPE "selectiondag" |
--- |
99 |
#define DEBUG_TYPE "selectiondag" |
--- |
| 100 |
|
--- |
100 |
|
--- |
| 101 |
static cl::opt EnableMemCpyDAGOpt("enable-memcpy-dag-opt", |
--- |
101 |
static cl::opt EnableMemCpyDAGOpt("enable-memcpy-dag-opt", |
--- |
| 102 |
cl::Hidden, cl::init(true), |
--- |
102 |
cl::Hidden, cl::init(true), |
--- |
| 103 |
cl::desc("Gang up loads and stores generated by inlining of memcpy")); |
--- |
103 |
cl::desc("Gang up loads and stores generated by inlining of memcpy")); |
--- |
| 104 |
|
--- |
104 |
|
--- |
| 105 |
static cl::opt MaxLdStGlue("ldstmemcpy-glue-max", |
--- |
105 |
static cl::opt MaxLdStGlue("ldstmemcpy-glue-max", |
--- |
| 106 |
cl::desc("Number limit for gluing ld/st of memcpy."), |
--- |
106 |
cl::desc("Number limit for gluing ld/st of memcpy."), |
--- |
| 107 |
cl::Hidden, cl::init(0)); |
--- |
107 |
cl::Hidden, cl::init(0)); |
--- |
| 108 |
|
--- |
108 |
|
--- |
| 109 |
static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) { |
39 |
109 |
static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) { |
39 |
| 110 |
LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G);); |
39 |
110 |
LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G);); |
39 |
| 111 |
} |
39 |
111 |
} |
39 |
| 112 |
|
--- |
112 |
|
--- |
| 113 |
//===----------------------------------------------------------------------===// |
--- |
113 |
//===----------------------------------------------------------------------===// |
--- |
| 114 |
// ConstantFPSDNode Class |
--- |
114 |
// ConstantFPSDNode Class |
--- |
| 115 |
//===----------------------------------------------------------------------===// |
--- |
115 |
//===----------------------------------------------------------------------===// |
--- |
| 116 |
|
--- |
116 |
|
--- |
| 117 |
/// isExactlyValue - We don't rely on operator== working on double values, as |
--- |
117 |
/// isExactlyValue - We don't rely on operator== working on double values, as |
--- |
| 118 |
/// it returns true for things that are clearly not equal, like -0.0 and 0.0. |
--- |
118 |
/// it returns true for things that are clearly not equal, like -0.0 and 0.0. |
--- |
| 119 |
/// As such, this method can be used to do an exact bit-for-bit comparison of |
--- |
119 |
/// As such, this method can be used to do an exact bit-for-bit comparison of |
--- |
| 120 |
/// two floating point values. |
--- |
120 |
/// two floating point values. |
--- |
| 121 |
bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { |
0 |
121 |
bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { |
0 |
| 122 |
return getValueAPF().bitwiseIsEqual(V); |
0 |
122 |
return getValueAPF().bitwiseIsEqual(V); |
0 |
| 123 |
} |
--- |
123 |
} |
--- |
| 124 |
|
--- |
124 |
|
--- |
| 125 |
bool ConstantFPSDNode::isValueValidForType(EVT VT, |
0 |
125 |
bool ConstantFPSDNode::isValueValidForType(EVT VT, |
0 |
| 126 |
const APFloat& Val) { |
--- |
126 |
const APFloat& Val) { |
--- |
| 127 |
assert(VT.isFloatingPoint() && "Can only convert between FP types"); |
0 |
127 |
assert(VT.isFloatingPoint() && "Can only convert between FP types"); |
0 |
| 128 |
|
--- |
128 |
|
--- |
| 129 |
// convert modifies in place, so make a copy. |
--- |
129 |
// convert modifies in place, so make a copy. |
--- |
| 130 |
APFloat Val2 = APFloat(Val); |
0 |
130 |
APFloat Val2 = APFloat(Val); |
0 |
| 131 |
bool losesInfo; |
--- |
131 |
bool losesInfo; |
--- |
| 132 |
(void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT), |
0 |
132 |
(void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT), |
0 |
| 133 |
APFloat::rmNearestTiesToEven, |
--- |
133 |
APFloat::rmNearestTiesToEven, |
--- |
| 134 |
&losesInfo); |
--- |
134 |
&losesInfo); |
--- |
| 135 |
return !losesInfo; |
0 |
135 |
return !losesInfo; |
0 |
| 136 |
} |
0 |
136 |
} |
0 |
| 137 |
|
--- |
137 |
|
--- |
| 138 |
//===----------------------------------------------------------------------===// |
--- |
138 |
//===----------------------------------------------------------------------===// |
--- |
| 139 |
// ISD Namespace |
--- |
139 |
// ISD Namespace |
--- |
| 140 |
//===----------------------------------------------------------------------===// |
--- |
140 |
//===----------------------------------------------------------------------===// |
--- |
| 141 |
|
--- |
141 |
|
--- |
| 142 |
bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) { |
0 |
142 |
bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) { |
0 |
| 143 |
if (N->getOpcode() == ISD::SPLAT_VECTOR) { |
0 |
143 |
if (N->getOpcode() == ISD::SPLAT_VECTOR) { |
0 |
| 144 |
unsigned EltSize = |
--- |
144 |
unsigned EltSize = |
--- |
| 145 |
N->getValueType(0).getVectorElementType().getSizeInBits(); |
0 |
145 |
N->getValueType(0).getVectorElementType().getSizeInBits(); |
0 |
| 146 |
if (auto *Op0 = dyn_cast(N->getOperand(0))) { |
0 |
146 |
if (auto *Op0 = dyn_cast(N->getOperand(0))) { |
0 |
| 147 |
SplatVal = Op0->getAPIntValue().trunc(EltSize); |
0 |
147 |
SplatVal = Op0->getAPIntValue().trunc(EltSize); |
0 |
| 148 |
return true; |
0 |
148 |
return true; |
0 |
| 149 |
} |
--- |
149 |
} |
--- |
| 150 |
if (auto *Op0 = dyn_cast(N->getOperand(0))) { |
0 |
150 |
if (auto *Op0 = dyn_cast(N->getOperand(0))) { |
0 |
| 151 |
SplatVal = Op0->getValueAPF().bitcastToAPInt().trunc(EltSize); |
0 |
151 |
SplatVal = Op0->getValueAPF().bitcastToAPInt().trunc(EltSize); |
0 |
| 152 |
return true; |
0 |
152 |
return true; |
0 |
| 153 |
} |
--- |
153 |
} |
--- |
| 154 |
} |
--- |
154 |
} |
--- |
| 155 |
|
--- |
155 |
|
--- |
| 156 |
auto *BV = dyn_cast(N); |
0 |
156 |
auto *BV = dyn_cast(N); |
0 |
| 157 |
if (!BV) |
0 |
157 |
if (!BV) |
0 |
| 158 |
return false; |
0 |
158 |
return false; |
0 |
| 159 |
|
--- |
159 |
|
--- |
| 160 |
APInt SplatUndef; |
0 |
160 |
APInt SplatUndef; |
0 |
| 161 |
unsigned SplatBitSize; |
--- |
161 |
unsigned SplatBitSize; |
--- |
| 162 |
bool HasUndefs; |
--- |
162 |
bool HasUndefs; |
--- |
| 163 |
unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits(); |
0 |
163 |
unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits(); |
0 |
| 164 |
return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs, |
0 |
164 |
return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs, |
0 |
| 165 |
EltSize) && |
0 |
165 |
EltSize) && |
0 |
| 166 |
EltSize == SplatBitSize; |
0 |
166 |
EltSize == SplatBitSize; |
0 |
| 167 |
} |
0 |
167 |
} |
0 |
| 168 |
|
--- |
168 |
|
--- |
| 169 |
// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be |
--- |
169 |
// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be |
--- |
| 170 |
// specializations of the more general isConstantSplatVector()? |
--- |
170 |
// specializations of the more general isConstantSplatVector()? |
--- |
| 171 |
|
--- |
171 |
|
--- |
| 172 |
bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) { |
0 |
172 |
bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) { |
0 |
| 173 |
// Look through a bit convert. |
--- |
173 |
// Look through a bit convert. |
--- |
| 174 |
while (N->getOpcode() == ISD::BITCAST) |
0 |
174 |
while (N->getOpcode() == ISD::BITCAST) |
0 |
| 175 |
N = N->getOperand(0).getNode(); |
0 |
175 |
N = N->getOperand(0).getNode(); |
0 |
| 176 |
|
--- |
176 |
|
--- |
| 177 |
if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { |
0 |
177 |
if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { |
0 |
| 178 |
APInt SplatVal; |
0 |
178 |
APInt SplatVal; |
0 |
| 179 |
return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes(); |
0 |
179 |
return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes(); |
0 |
| 180 |
} |
0 |
180 |
} |
0 |
| 181 |
|
--- |
181 |
|
--- |
| 182 |
if (N->getOpcode() != ISD::BUILD_VECTOR) return false; |
0 |
182 |
if (N->getOpcode() != ISD::BUILD_VECTOR) return false; |
0 |
| 183 |
|
--- |
183 |
|
--- |
| 184 |
unsigned i = 0, e = N->getNumOperands(); |
0 |
184 |
unsigned i = 0, e = N->getNumOperands(); |
0 |
| 185 |
|
--- |
185 |
|
--- |
| 186 |
// Skip over all of the undef values. |
--- |
186 |
// Skip over all of the undef values. |
--- |
| 187 |
while (i != e && N->getOperand(i).isUndef()) |
0 |
187 |
while (i != e && N->getOperand(i).isUndef()) |
0 |
| 188 |
++i; |
0 |
188 |
++i; |
0 |
| 189 |
|
--- |
189 |
|
--- |
| 190 |
// Do not accept an all-undef vector. |
--- |
190 |
// Do not accept an all-undef vector. |
--- |
| 191 |
if (i == e) return false; |
0 |
191 |
if (i == e) return false; |
0 |
| 192 |
|
--- |
192 |
|
--- |
| 193 |
// Do not accept build_vectors that aren't all constants or which have non-~0 |
--- |
193 |
// Do not accept build_vectors that aren't all constants or which have non-~0 |
--- |
| 194 |
// elements. We have to be a bit careful here, as the type of the constant |
--- |
194 |
// elements. We have to be a bit careful here, as the type of the constant |
--- |
| 195 |
// may not be the same as the type of the vector elements due to type |
--- |
195 |
// may not be the same as the type of the vector elements due to type |
--- |
| 196 |
// legalization (the elements are promoted to a legal type for the target and |
--- |
196 |
// legalization (the elements are promoted to a legal type for the target and |
--- |
| 197 |
// a vector of a type may be legal when the base element type is not). |
--- |
197 |
// a vector of a type may be legal when the base element type is not). |
--- |
| 198 |
// We only want to check enough bits to cover the vector elements, because |
--- |
198 |
// We only want to check enough bits to cover the vector elements, because |
--- |
| 199 |
// we care if the resultant vector is all ones, not whether the individual |
--- |
199 |
// we care if the resultant vector is all ones, not whether the individual |
--- |
| 200 |
// constants are. |
--- |
200 |
// constants are. |
--- |
| 201 |
SDValue NotZero = N->getOperand(i); |
0 |
201 |
SDValue NotZero = N->getOperand(i); |
0 |
| 202 |
unsigned EltSize = N->getValueType(0).getScalarSizeInBits(); |
0 |
202 |
unsigned EltSize = N->getValueType(0).getScalarSizeInBits(); |
0 |
| 203 |
if (ConstantSDNode *CN = dyn_cast(NotZero)) { |
0 |
203 |
if (ConstantSDNode *CN = dyn_cast(NotZero)) { |
0 |
| 204 |
if (CN->getAPIntValue().countr_one() < EltSize) |
0 |
204 |
if (CN->getAPIntValue().countr_one() < EltSize) |
0 |
| 205 |
return false; |
0 |
205 |
return false; |
0 |
| 206 |
} else if (ConstantFPSDNode *CFPN = dyn_cast(NotZero)) { |
0 |
206 |
} else if (ConstantFPSDNode *CFPN = dyn_cast(NotZero)) { |
0 |
| 207 |
if (CFPN->getValueAPF().bitcastToAPInt().countr_one() < EltSize) |
0 |
207 |
if (CFPN->getValueAPF().bitcastToAPInt().countr_one() < EltSize) |
0 |
| 208 |
return false; |
0 |
208 |
return false; |
0 |
| 209 |
} else |
--- |
209 |
} else |
--- |
| 210 |
return false; |
0 |
210 |
return false; |
0 |
| 211 |
|
--- |
211 |
|
--- |
| 212 |
// Okay, we have at least one ~0 value, check to see if the rest match or are |
--- |
212 |
// Okay, we have at least one ~0 value, check to see if the rest match or are |
--- |
| 213 |
// undefs. Even with the above element type twiddling, this should be OK, as |
--- |
213 |
// undefs. Even with the above element type twiddling, this should be OK, as |
--- |
| 214 |
// the same type legalization should have applied to all the elements. |
--- |
214 |
// the same type legalization should have applied to all the elements. |
--- |
| 215 |
for (++i; i != e; ++i) |
0 |
215 |
for (++i; i != e; ++i) |
0 |
| 216 |
if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef()) |
0 |
216 |
if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef()) |
0 |
| 217 |
return false; |
0 |
217 |
return false; |
0 |
| 218 |
return true; |
0 |
218 |
return true; |
0 |
| 219 |
} |
--- |
219 |
} |
--- |
| 220 |
|
--- |
220 |
|
--- |
| 221 |
bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) { |
0 |
221 |
bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) { |
0 |
| 222 |
// Look through a bit convert. |
--- |
222 |
// Look through a bit convert. |
--- |
| 223 |
while (N->getOpcode() == ISD::BITCAST) |
0 |
223 |
while (N->getOpcode() == ISD::BITCAST) |
0 |
| 224 |
N = N->getOperand(0).getNode(); |
0 |
224 |
N = N->getOperand(0).getNode(); |
0 |
| 225 |
|
--- |
225 |
|
--- |
| 226 |
if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { |
0 |
226 |
if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { |
0 |
| 227 |
APInt SplatVal; |
0 |
227 |
APInt SplatVal; |
0 |
| 228 |
return isConstantSplatVector(N, SplatVal) && SplatVal.isZero(); |
0 |
228 |
return isConstantSplatVector(N, SplatVal) && SplatVal.isZero(); |
0 |
| 229 |
} |
0 |
229 |
} |
0 |
| 230 |
|
--- |
230 |
|
--- |
| 231 |
if (N->getOpcode() != ISD::BUILD_VECTOR) return false; |
0 |
231 |
if (N->getOpcode() != ISD::BUILD_VECTOR) return false; |
0 |
| 232 |
|
--- |
232 |
|
--- |
| 233 |
bool IsAllUndef = true; |
0 |
233 |
bool IsAllUndef = true; |
0 |
| 234 |
for (const SDValue &Op : N->op_values()) { |
0 |
234 |
for (const SDValue &Op : N->op_values()) { |
0 |
| 235 |
if (Op.isUndef()) |
0 |
235 |
if (Op.isUndef()) |
0 |
| 236 |
continue; |
0 |
236 |
continue; |
0 |
| 237 |
IsAllUndef = false; |
0 |
237 |
IsAllUndef = false; |
0 |
| 238 |
// Do not accept build_vectors that aren't all constants or which have non-0 |
--- |
238 |
// Do not accept build_vectors that aren't all constants or which have non-0 |
--- |
| 239 |
// elements. We have to be a bit careful here, as the type of the constant |
--- |
239 |
// elements. We have to be a bit careful here, as the type of the constant |
--- |
| 240 |
// may not be the same as the type of the vector elements due to type |
--- |
240 |
// may not be the same as the type of the vector elements due to type |
--- |
| 241 |
// legalization (the elements are promoted to a legal type for the target |
--- |
241 |
// legalization (the elements are promoted to a legal type for the target |
--- |
| 242 |
// and a vector of a type may be legal when the base element type is not). |
--- |
242 |
// and a vector of a type may be legal when the base element type is not). |
--- |
| 243 |
// We only want to check enough bits to cover the vector elements, because |
--- |
243 |
// We only want to check enough bits to cover the vector elements, because |
--- |
| 244 |
// we care if the resultant vector is all zeros, not whether the individual |
--- |
244 |
// we care if the resultant vector is all zeros, not whether the individual |
--- |
| 245 |
// constants are. |
--- |
245 |
// constants are. |
--- |
| 246 |
unsigned EltSize = N->getValueType(0).getScalarSizeInBits(); |
0 |
246 |
unsigned EltSize = N->getValueType(0).getScalarSizeInBits(); |
0 |
| 247 |
if (ConstantSDNode *CN = dyn_cast(Op)) { |
0 |
247 |
if (ConstantSDNode *CN = dyn_cast(Op)) { |
0 |
| 248 |
if (CN->getAPIntValue().countr_zero() < EltSize) |
0 |
248 |
if (CN->getAPIntValue().countr_zero() < EltSize) |
0 |
| 249 |
return false; |
0 |
249 |
return false; |
0 |
| 250 |
} else if (ConstantFPSDNode *CFPN = dyn_cast(Op)) { |
0 |
250 |
} else if (ConstantFPSDNode *CFPN = dyn_cast(Op)) { |
0 |
| 251 |
if (CFPN->getValueAPF().bitcastToAPInt().countr_zero() < EltSize) |
0 |
251 |
if (CFPN->getValueAPF().bitcastToAPInt().countr_zero() < EltSize) |
0 |
| 252 |
return false; |
0 |
252 |
return false; |
0 |
| 253 |
} else |
--- |
253 |
} else |
--- |
| 254 |
return false; |
0 |
254 |
return false; |
0 |
| 255 |
} |
--- |
255 |
} |
--- |
| 256 |
|
--- |
256 |
|
--- |
| 257 |
// Do not accept an all-undef vector. |
--- |
257 |
// Do not accept an all-undef vector. |
--- |
| 258 |
if (IsAllUndef) |
0 |
258 |
if (IsAllUndef) |
0 |
| 259 |
return false; |
0 |
259 |
return false; |
0 |
| 260 |
return true; |
0 |
260 |
return true; |
0 |
| 261 |
} |
--- |
261 |
} |
--- |
| 262 |
|
--- |
262 |
|
--- |
| 263 |
bool ISD::isBuildVectorAllOnes(const SDNode *N) { |
0 |
263 |
bool ISD::isBuildVectorAllOnes(const SDNode *N) { |
0 |
| 264 |
return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true); |
0 |
264 |
return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true); |
0 |
| 265 |
} |
--- |
265 |
} |
--- |
| 266 |
|
--- |
266 |
|
--- |
| 267 |
bool ISD::isBuildVectorAllZeros(const SDNode *N) { |
0 |
267 |
bool ISD::isBuildVectorAllZeros(const SDNode *N) { |
0 |
| 268 |
return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true); |
0 |
268 |
return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true); |
0 |
| 269 |
} |
--- |
269 |
} |
--- |
| 270 |
|
--- |
270 |
|
--- |
| 271 |
bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) { |
14 |
271 |
bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) { |
14 |
| 272 |
if (N->getOpcode() != ISD::BUILD_VECTOR) |
14 |
272 |
if (N->getOpcode() != ISD::BUILD_VECTOR) |
14 |
| 273 |
return false; |
14 |
273 |
return false; |
14 |
| 274 |
|
--- |
274 |
|
--- |
| 275 |
for (const SDValue &Op : N->op_values()) { |
0 |
275 |
for (const SDValue &Op : N->op_values()) { |
0 |
| 276 |
if (Op.isUndef()) |
0 |
276 |
if (Op.isUndef()) |
0 |
| 277 |
continue; |
0 |
277 |
continue; |
0 |
| 278 |
if (!isa(Op)) |
0 |
278 |
if (!isa(Op)) |
0 |
| 279 |
return false; |
0 |
279 |
return false; |
0 |
| 280 |
} |
--- |
280 |
} |
--- |
| 281 |
return true; |
0 |
281 |
return true; |
0 |
| 282 |
} |
--- |
282 |
} |
--- |
| 283 |
|
--- |
283 |
|
--- |
| 284 |
bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) { |
22 |
284 |
bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) { |
22 |
| 285 |
if (N->getOpcode() != ISD::BUILD_VECTOR) |
22 |
285 |
if (N->getOpcode() != ISD::BUILD_VECTOR) |
22 |
| 286 |
return false; |
22 |
286 |
return false; |
22 |
| 287 |
|
--- |
287 |
|
--- |
| 288 |
for (const SDValue &Op : N->op_values()) { |
0 |
288 |
for (const SDValue &Op : N->op_values()) { |
0 |
| 289 |
if (Op.isUndef()) |
0 |
289 |
if (Op.isUndef()) |
0 |
| 290 |
continue; |
0 |
290 |
continue; |
0 |
| 291 |
if (!isa(Op)) |
0 |
291 |
if (!isa(Op)) |
0 |
| 292 |
return false; |
0 |
292 |
return false; |
0 |
| 293 |
} |
--- |
293 |
} |
--- |
| 294 |
return true; |
0 |
294 |
return true; |
0 |
| 295 |
} |
--- |
295 |
} |
--- |
| 296 |
|
--- |
296 |
|
--- |
| 297 |
bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize, |
0 |
297 |
bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize, |
0 |
| 298 |
bool Signed) { |
--- |
298 |
bool Signed) { |
--- |
| 299 |
assert(N->getValueType(0).isVector() && "Expected a vector!"); |
0 |
299 |
assert(N->getValueType(0).isVector() && "Expected a vector!"); |
0 |
| 300 |
|
--- |
300 |
|
--- |
| 301 |
unsigned EltSize = N->getValueType(0).getScalarSizeInBits(); |
0 |
301 |
unsigned EltSize = N->getValueType(0).getScalarSizeInBits(); |
0 |
| 302 |
if (EltSize <= NewEltSize) |
0 |
302 |
if (EltSize <= NewEltSize) |
0 |
| 303 |
return false; |
0 |
303 |
return false; |
0 |
| 304 |
|
--- |
304 |
|
--- |
| 305 |
if (N->getOpcode() == ISD::ZERO_EXTEND) { |
0 |
305 |
if (N->getOpcode() == ISD::ZERO_EXTEND) { |
0 |
| 306 |
return (N->getOperand(0).getValueType().getScalarSizeInBits() <= |
0 |
306 |
return (N->getOperand(0).getValueType().getScalarSizeInBits() <= |
0 |
| 307 |
NewEltSize) && |
0 |
307 |
NewEltSize) && |
0 |
| 308 |
!Signed; |
0 |
308 |
!Signed; |
0 |
| 309 |
} |
--- |
309 |
} |
--- |
| 310 |
if (N->getOpcode() == ISD::SIGN_EXTEND) { |
0 |
310 |
if (N->getOpcode() == ISD::SIGN_EXTEND) { |
0 |
| 311 |
return (N->getOperand(0).getValueType().getScalarSizeInBits() <= |
0 |
311 |
return (N->getOperand(0).getValueType().getScalarSizeInBits() <= |
0 |
| 312 |
NewEltSize) && |
0 |
312 |
NewEltSize) && |
0 |
| 313 |
Signed; |
0 |
313 |
Signed; |
0 |
| 314 |
} |
--- |
314 |
} |
--- |
| 315 |
if (N->getOpcode() != ISD::BUILD_VECTOR) |
0 |
315 |
if (N->getOpcode() != ISD::BUILD_VECTOR) |
0 |
| 316 |
return false; |
0 |
316 |
return false; |
0 |
| 317 |
|
--- |
317 |
|
--- |
| 318 |
for (const SDValue &Op : N->op_values()) { |
0 |
318 |
for (const SDValue &Op : N->op_values()) { |
0 |
| 319 |
if (Op.isUndef()) |
0 |
319 |
if (Op.isUndef()) |
0 |
| 320 |
continue; |
0 |
320 |
continue; |
0 |
| 321 |
if (!isa(Op)) |
0 |
321 |
if (!isa(Op)) |
0 |
| 322 |
return false; |
0 |
322 |
return false; |
0 |
| 323 |
|
--- |
323 |
|
--- |
| 324 |
APInt C = cast(Op)->getAPIntValue().trunc(EltSize); |
0 |
324 |
APInt C = cast(Op)->getAPIntValue().trunc(EltSize); |
0 |
| 325 |
if (Signed && C.trunc(NewEltSize).sext(EltSize) != C) |
0 |
325 |
if (Signed && C.trunc(NewEltSize).sext(EltSize) != C) |
0 |
| 326 |
return false; |
0 |
326 |
return false; |
0 |
| 327 |
if (!Signed && C.trunc(NewEltSize).zext(EltSize) != C) |
0 |
327 |
if (!Signed && C.trunc(NewEltSize).zext(EltSize) != C) |
0 |
| 328 |
return false; |
0 |
328 |
return false; |
0 |
| 329 |
} |
0 |
329 |
} |
0 |
| 330 |
|
--- |
330 |
|
--- |
| 331 |
return true; |
0 |
331 |
return true; |
0 |
| 332 |
} |
--- |
332 |
} |
--- |
| 333 |
|
--- |
333 |
|
--- |
| 334 |
bool ISD::allOperandsUndef(const SDNode *N) { |
0 |
334 |
bool ISD::allOperandsUndef(const SDNode *N) { |
0 |
| 335 |
// Return false if the node has no operands. |
--- |
335 |
// Return false if the node has no operands. |
--- |
| 336 |
// This is "logically inconsistent" with the definition of "all" but |
--- |
336 |
// This is "logically inconsistent" with the definition of "all" but |
--- |
| 337 |
// is probably the desired behavior. |
--- |
337 |
// is probably the desired behavior. |
--- |
| 338 |
if (N->getNumOperands() == 0) |
0 |
338 |
if (N->getNumOperands() == 0) |
0 |
| 339 |
return false; |
0 |
339 |
return false; |
0 |
| 340 |
return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); }); |
0 |
340 |
return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); }); |
0 |
| 341 |
} |
--- |
341 |
} |
--- |
| 342 |
|
--- |
342 |
|
--- |
| 343 |
bool ISD::isFreezeUndef(const SDNode *N) { |
0 |
343 |
bool ISD::isFreezeUndef(const SDNode *N) { |
0 |
| 344 |
return N->getOpcode() == ISD::FREEZE && N->getOperand(0).isUndef(); |
0 |
344 |
return N->getOpcode() == ISD::FREEZE && N->getOperand(0).isUndef(); |
0 |
| 345 |
} |
--- |
345 |
} |
--- |
| 346 |
|
--- |
346 |
|
--- |
| 347 |
bool ISD::matchUnaryPredicate(SDValue Op, |
0 |
347 |
bool ISD::matchUnaryPredicate(SDValue Op, |
0 |
| 348 |
std::function Match, |
--- |
348 |
std::function Match, |
--- |
| 349 |
bool AllowUndefs) { |
--- |
349 |
bool AllowUndefs) { |
--- |
| 350 |
// FIXME: Add support for scalar UNDEF cases? |
--- |
350 |
// FIXME: Add support for scalar UNDEF cases? |
--- |
| 351 |
if (auto *Cst = dyn_cast(Op)) |
0 |
351 |
if (auto *Cst = dyn_cast(Op)) |
0 |
| 352 |
return Match(Cst); |
0 |
352 |
return Match(Cst); |
0 |
| 353 |
|
--- |
353 |
|
--- |
| 354 |
// FIXME: Add support for vector UNDEF cases? |
--- |
354 |
// FIXME: Add support for vector UNDEF cases? |
--- |
| 355 |
if (ISD::BUILD_VECTOR != Op.getOpcode() && |
0 |
355 |
if (ISD::BUILD_VECTOR != Op.getOpcode() && |
0 |
| 356 |
ISD::SPLAT_VECTOR != Op.getOpcode()) |
0 |
356 |
ISD::SPLAT_VECTOR != Op.getOpcode()) |
0 |
| 357 |
return false; |
0 |
357 |
return false; |
0 |
| 358 |
|
--- |
358 |
|
--- |
| 359 |
EVT SVT = Op.getValueType().getScalarType(); |
0 |
359 |
EVT SVT = Op.getValueType().getScalarType(); |
0 |
| 360 |
for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) { |
0 |
360 |
for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) { |
0 |
| 361 |
if (AllowUndefs && Op.getOperand(i).isUndef()) { |
0 |
361 |
if (AllowUndefs && Op.getOperand(i).isUndef()) { |
0 |
| 362 |
if (!Match(nullptr)) |
0 |
362 |
if (!Match(nullptr)) |
0 |
| 363 |
return false; |
0 |
363 |
return false; |
0 |
| 364 |
continue; |
0 |
364 |
continue; |
0 |
| 365 |
} |
--- |
365 |
} |
--- |
| 366 |
|
--- |
366 |
|
--- |
| 367 |
auto *Cst = dyn_cast(Op.getOperand(i)); |
0 |
367 |
auto *Cst = dyn_cast(Op.getOperand(i)); |
0 |
| 368 |
if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst)) |
0 |
368 |
if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst)) |
0 |
| 369 |
return false; |
0 |
369 |
return false; |
0 |
| 370 |
} |
--- |
370 |
} |
--- |
| 371 |
return true; |
0 |
371 |
return true; |
0 |
| 372 |
} |
--- |
372 |
} |
--- |
| 373 |
|
--- |
373 |
|
--- |
| 374 |
bool ISD::matchBinaryPredicate( |
0 |
374 |
bool ISD::matchBinaryPredicate( |
0 |
| 375 |
SDValue LHS, SDValue RHS, |
--- |
375 |
SDValue LHS, SDValue RHS, |
--- |
| 376 |
std::function Match, |
--- |
376 |
std::function Match, |
--- |
| 377 |
bool AllowUndefs, bool AllowTypeMismatch) { |
--- |
377 |
bool AllowUndefs, bool AllowTypeMismatch) { |
--- |
| 378 |
if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType()) |
0 |
378 |
if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType()) |
0 |
| 379 |
return false; |
0 |
379 |
return false; |
0 |
| 380 |
|
--- |
380 |
|
--- |
| 381 |
// TODO: Add support for scalar UNDEF cases? |
--- |
381 |
// TODO: Add support for scalar UNDEF cases? |
--- |
| 382 |
if (auto *LHSCst = dyn_cast(LHS)) |
0 |
382 |
if (auto *LHSCst = dyn_cast(LHS)) |
0 |
| 383 |
if (auto *RHSCst = dyn_cast(RHS)) |
0 |
383 |
if (auto *RHSCst = dyn_cast(RHS)) |
0 |
| 384 |
return Match(LHSCst, RHSCst); |
0 |
384 |
return Match(LHSCst, RHSCst); |
0 |
| 385 |
|
--- |
385 |
|
--- |
| 386 |
// TODO: Add support for vector UNDEF cases? |
--- |
386 |
// TODO: Add support for vector UNDEF cases? |
--- |
| 387 |
if (LHS.getOpcode() != RHS.getOpcode() || |
0 |
387 |
if (LHS.getOpcode() != RHS.getOpcode() || |
0 |
| 388 |
(LHS.getOpcode() != ISD::BUILD_VECTOR && |
0 |
388 |
(LHS.getOpcode() != ISD::BUILD_VECTOR && |
0 |
| 389 |
LHS.getOpcode() != ISD::SPLAT_VECTOR)) |
0 |
389 |
LHS.getOpcode() != ISD::SPLAT_VECTOR)) |
0 |
| 390 |
return false; |
0 |
390 |
return false; |
0 |
| 391 |
|
--- |
391 |
|
--- |
| 392 |
EVT SVT = LHS.getValueType().getScalarType(); |
0 |
392 |
EVT SVT = LHS.getValueType().getScalarType(); |
0 |
| 393 |
for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) { |
0 |
393 |
for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) { |
0 |
| 394 |
SDValue LHSOp = LHS.getOperand(i); |
0 |
394 |
SDValue LHSOp = LHS.getOperand(i); |
0 |
| 395 |
SDValue RHSOp = RHS.getOperand(i); |
0 |
395 |
SDValue RHSOp = RHS.getOperand(i); |
0 |
| 396 |
bool LHSUndef = AllowUndefs && LHSOp.isUndef(); |
0 |
396 |
bool LHSUndef = AllowUndefs && LHSOp.isUndef(); |
0 |
| 397 |
bool RHSUndef = AllowUndefs && RHSOp.isUndef(); |
0 |
397 |
bool RHSUndef = AllowUndefs && RHSOp.isUndef(); |
0 |
| 398 |
auto *LHSCst = dyn_cast(LHSOp); |
0 |
398 |
auto *LHSCst = dyn_cast(LHSOp); |
0 |
| 399 |
auto *RHSCst = dyn_cast(RHSOp); |
0 |
399 |
auto *RHSCst = dyn_cast(RHSOp); |
0 |
| 400 |
if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef)) |
0 |
400 |
if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef)) |
0 |
| 401 |
return false; |
0 |
401 |
return false; |
0 |
| 402 |
if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT || |
0 |
402 |
if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT || |
0 |
| 403 |
LHSOp.getValueType() != RHSOp.getValueType())) |
0 |
403 |
LHSOp.getValueType() != RHSOp.getValueType())) |
0 |
| 404 |
return false; |
0 |
404 |
return false; |
0 |
| 405 |
if (!Match(LHSCst, RHSCst)) |
0 |
405 |
if (!Match(LHSCst, RHSCst)) |
0 |
| 406 |
return false; |
0 |
406 |
return false; |
0 |
| 407 |
} |
--- |
407 |
} |
--- |
| 408 |
return true; |
0 |
408 |
return true; |
0 |
| 409 |
} |
--- |
409 |
} |
--- |
| 410 |
|
--- |
410 |
|
--- |
| 411 |
ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) { |
0 |
411 |
ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) { |
0 |
| 412 |
switch (VecReduceOpcode) { |
0 |
412 |
switch (VecReduceOpcode) { |
0 |
| 413 |
default: |
0 |
413 |
default: |
0 |
| 414 |
llvm_unreachable("Expected VECREDUCE opcode"); |
0 |
414 |
llvm_unreachable("Expected VECREDUCE opcode"); |
0 |
| 415 |
case ISD::VECREDUCE_FADD: |
0 |
415 |
case ISD::VECREDUCE_FADD: |
0 |
| 416 |
case ISD::VECREDUCE_SEQ_FADD: |
--- |
416 |
case ISD::VECREDUCE_SEQ_FADD: |
--- |
| 417 |
case ISD::VP_REDUCE_FADD: |
--- |
417 |
case ISD::VP_REDUCE_FADD: |
--- |
| 418 |
case ISD::VP_REDUCE_SEQ_FADD: |
--- |
418 |
case ISD::VP_REDUCE_SEQ_FADD: |
--- |
| 419 |
return ISD::FADD; |
0 |
419 |
return ISD::FADD; |
0 |
| 420 |
case ISD::VECREDUCE_FMUL: |
0 |
420 |
case ISD::VECREDUCE_FMUL: |
0 |
| 421 |
case ISD::VECREDUCE_SEQ_FMUL: |
--- |
421 |
case ISD::VECREDUCE_SEQ_FMUL: |
--- |
| 422 |
case ISD::VP_REDUCE_FMUL: |
--- |
422 |
case ISD::VP_REDUCE_FMUL: |
--- |
| 423 |
case ISD::VP_REDUCE_SEQ_FMUL: |
--- |
423 |
case ISD::VP_REDUCE_SEQ_FMUL: |
--- |
| 424 |
return ISD::FMUL; |
0 |
424 |
return ISD::FMUL; |
0 |
| 425 |
case ISD::VECREDUCE_ADD: |
0 |
425 |
case ISD::VECREDUCE_ADD: |
0 |
| 426 |
case ISD::VP_REDUCE_ADD: |
--- |
426 |
case ISD::VP_REDUCE_ADD: |
--- |
| 427 |
return ISD::ADD; |
0 |
427 |
return ISD::ADD; |
0 |
| 428 |
case ISD::VECREDUCE_MUL: |
0 |
428 |
case ISD::VECREDUCE_MUL: |
0 |
| 429 |
case ISD::VP_REDUCE_MUL: |
--- |
429 |
case ISD::VP_REDUCE_MUL: |
--- |
| 430 |
return ISD::MUL; |
0 |
430 |
return ISD::MUL; |
0 |
| 431 |
case ISD::VECREDUCE_AND: |
0 |
431 |
case ISD::VECREDUCE_AND: |
0 |
| 432 |
case ISD::VP_REDUCE_AND: |
--- |
432 |
case ISD::VP_REDUCE_AND: |
--- |
| 433 |
return ISD::AND; |
0 |
433 |
return ISD::AND; |
0 |
| 434 |
case ISD::VECREDUCE_OR: |
0 |
434 |
case ISD::VECREDUCE_OR: |
0 |
| 435 |
case ISD::VP_REDUCE_OR: |
--- |
435 |
case ISD::VP_REDUCE_OR: |
--- |
| 436 |
return ISD::OR; |
0 |
436 |
return ISD::OR; |
0 |
| 437 |
case ISD::VECREDUCE_XOR: |
0 |
437 |
case ISD::VECREDUCE_XOR: |
0 |
| 438 |
case ISD::VP_REDUCE_XOR: |
--- |
438 |
case ISD::VP_REDUCE_XOR: |
--- |
| 439 |
return ISD::XOR; |
0 |
439 |
return ISD::XOR; |
0 |
| 440 |
case ISD::VECREDUCE_SMAX: |
0 |
440 |
case ISD::VECREDUCE_SMAX: |
0 |
| 441 |
case ISD::VP_REDUCE_SMAX: |
--- |
441 |
case ISD::VP_REDUCE_SMAX: |
--- |
| 442 |
return ISD::SMAX; |
0 |
442 |
return ISD::SMAX; |
0 |
| 443 |
case ISD::VECREDUCE_SMIN: |
0 |
443 |
case ISD::VECREDUCE_SMIN: |
0 |
| 444 |
case ISD::VP_REDUCE_SMIN: |
--- |
444 |
case ISD::VP_REDUCE_SMIN: |
--- |
| 445 |
return ISD::SMIN; |
0 |
445 |
return ISD::SMIN; |
0 |
| 446 |
case ISD::VECREDUCE_UMAX: |
0 |
446 |
case ISD::VECREDUCE_UMAX: |
0 |
| 447 |
case ISD::VP_REDUCE_UMAX: |
--- |
447 |
case ISD::VP_REDUCE_UMAX: |
--- |
| 448 |
return ISD::UMAX; |
0 |
448 |
return ISD::UMAX; |
0 |
| 449 |
case ISD::VECREDUCE_UMIN: |
0 |
449 |
case ISD::VECREDUCE_UMIN: |
0 |
| 450 |
case ISD::VP_REDUCE_UMIN: |
--- |
450 |
case ISD::VP_REDUCE_UMIN: |
--- |
| 451 |
return ISD::UMIN; |
0 |
451 |
return ISD::UMIN; |
0 |
| 452 |
case ISD::VECREDUCE_FMAX: |
0 |
452 |
case ISD::VECREDUCE_FMAX: |
0 |
| 453 |
case ISD::VP_REDUCE_FMAX: |
--- |
453 |
case ISD::VP_REDUCE_FMAX: |
--- |
| 454 |
return ISD::FMAXNUM; |
0 |
454 |
return ISD::FMAXNUM; |
0 |
| 455 |
case ISD::VECREDUCE_FMIN: |
0 |
455 |
case ISD::VECREDUCE_FMIN: |
0 |
| 456 |
case ISD::VP_REDUCE_FMIN: |
--- |
456 |
case ISD::VP_REDUCE_FMIN: |
--- |
| 457 |
return ISD::FMINNUM; |
0 |
457 |
return ISD::FMINNUM; |
0 |
| 458 |
case ISD::VECREDUCE_FMAXIMUM: |
0 |
458 |
case ISD::VECREDUCE_FMAXIMUM: |
0 |
| 459 |
return ISD::FMAXIMUM; |
0 |
459 |
return ISD::FMAXIMUM; |
0 |
| 460 |
case ISD::VECREDUCE_FMINIMUM: |
0 |
460 |
case ISD::VECREDUCE_FMINIMUM: |
0 |
| 461 |
return ISD::FMINIMUM; |
0 |
461 |
return ISD::FMINIMUM; |
0 |
| 462 |
} |
--- |
462 |
} |
--- |
| 463 |
} |
--- |
463 |
} |
--- |
| 464 |
|
--- |
464 |
|
--- |
| 465 |
bool ISD::isVPOpcode(unsigned Opcode) { |
0 |
465 |
bool ISD::isVPOpcode(unsigned Opcode) { |
0 |
| 466 |
switch (Opcode) { |
0 |
466 |
switch (Opcode) { |
0 |
| 467 |
default: |
0 |
467 |
default: |
0 |
| 468 |
return false; |
0 |
468 |
return false; |
0 |
| 469 |
#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \ |
--- |
469 |
#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \ |
--- |
| 470 |
case ISD::VPSD: \ |
--- |
470 |
case ISD::VPSD: \ |
--- |
| 471 |
return true; |
--- |
471 |
return true; |
--- |
| 472 |
#include "llvm/IR/VPIntrinsics.def" |
--- |
472 |
#include "llvm/IR/VPIntrinsics.def" |
--- |
| 473 |
} |
--- |
473 |
} |
--- |
| 474 |
} |
--- |
474 |
} |
--- |
| 475 |
|
--- |
475 |
|
--- |
| 476 |
bool ISD::isVPBinaryOp(unsigned Opcode) { |
0 |
476 |
bool ISD::isVPBinaryOp(unsigned Opcode) { |
0 |
| 477 |
switch (Opcode) { |
0 |
477 |
switch (Opcode) { |
0 |
| 478 |
default: |
0 |
478 |
default: |
0 |
| 479 |
break; |
0 |
479 |
break; |
0 |
| 480 |
#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD: |
--- |
480 |
#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD: |
--- |
| 481 |
#define VP_PROPERTY_BINARYOP return true; |
--- |
481 |
#define VP_PROPERTY_BINARYOP return true; |
--- |
| 482 |
#define END_REGISTER_VP_SDNODE(VPSD) break; |
--- |
482 |
#define END_REGISTER_VP_SDNODE(VPSD) break; |
--- |
| 483 |
#include "llvm/IR/VPIntrinsics.def" |
--- |
483 |
#include "llvm/IR/VPIntrinsics.def" |
--- |
| 484 |
} |
--- |
484 |
} |
--- |
| 485 |
return false; |
0 |
485 |
return false; |
0 |
| 486 |
} |
--- |
486 |
} |
--- |
| 487 |
|
--- |
487 |
|
--- |
| 488 |
bool ISD::isVPReduction(unsigned Opcode) { |
0 |
488 |
bool ISD::isVPReduction(unsigned Opcode) { |
0 |
| 489 |
switch (Opcode) { |
0 |
489 |
switch (Opcode) { |
0 |
| 490 |
default: |
0 |
490 |
default: |
0 |
| 491 |
break; |
0 |
491 |
break; |
0 |
| 492 |
#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD: |
--- |
492 |
#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD: |
--- |
| 493 |
#define VP_PROPERTY_REDUCTION(STARTPOS, ...) return true; |
--- |
493 |
#define VP_PROPERTY_REDUCTION(STARTPOS, ...) return true; |
--- |
| 494 |
#define END_REGISTER_VP_SDNODE(VPSD) break; |
--- |
494 |
#define END_REGISTER_VP_SDNODE(VPSD) break; |
--- |
| 495 |
#include "llvm/IR/VPIntrinsics.def" |
--- |
495 |
#include "llvm/IR/VPIntrinsics.def" |
--- |
| 496 |
} |
--- |
496 |
} |
--- |
| 497 |
return false; |
0 |
497 |
return false; |
0 |
| 498 |
} |
--- |
498 |
} |
--- |
| 499 |
|
--- |
499 |
|
--- |
| 500 |
/// The operand position of the vector mask. |
--- |
500 |
/// The operand position of the vector mask. |
--- |
| 501 |
std::optional ISD::getVPMaskIdx(unsigned Opcode) { |
0 |
501 |
std::optional ISD::getVPMaskIdx(unsigned Opcode) { |
0 |
| 502 |
switch (Opcode) { |
0 |
502 |
switch (Opcode) { |
0 |
| 503 |
default: |
0 |
503 |
default: |
0 |
| 504 |
return std::nullopt; |
0 |
504 |
return std::nullopt; |
0 |
| 505 |
#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \ |
--- |
505 |
#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \ |
--- |
| 506 |
case ISD::VPSD: \ |
--- |
506 |
case ISD::VPSD: \ |
--- |
| 507 |
return MASKPOS; |
--- |
507 |
return MASKPOS; |
--- |
| 508 |
#include "llvm/IR/VPIntrinsics.def" |
--- |
508 |
#include "llvm/IR/VPIntrinsics.def" |
--- |
| 509 |
} |
--- |
509 |
} |
--- |
| 510 |
} |
--- |
510 |
} |
--- |
| 511 |
|
--- |
511 |
|
--- |
| 512 |
/// The operand position of the explicit vector length parameter. |
--- |
512 |
/// The operand position of the explicit vector length parameter. |
--- |
| 513 |
std::optional ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) { |
0 |
513 |
std::optional ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) { |
0 |
| 514 |
switch (Opcode) { |
0 |
514 |
switch (Opcode) { |
0 |
| 515 |
default: |
0 |
515 |
default: |
0 |
| 516 |
return std::nullopt; |
0 |
516 |
return std::nullopt; |
0 |
| 517 |
#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \ |
--- |
517 |
#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \ |
--- |
| 518 |
case ISD::VPSD: \ |
--- |
518 |
case ISD::VPSD: \ |
--- |
| 519 |
return EVLPOS; |
--- |
519 |
return EVLPOS; |
--- |
| 520 |
#include "llvm/IR/VPIntrinsics.def" |
--- |
520 |
#include "llvm/IR/VPIntrinsics.def" |
--- |
| 521 |
} |
--- |
521 |
} |
--- |
| 522 |
} |
--- |
522 |
} |
--- |
| 523 |
|
--- |
523 |
|
--- |
| 524 |
std::optional ISD::getBaseOpcodeForVP(unsigned VPOpcode, |
0 |
524 |
std::optional ISD::getBaseOpcodeForVP(unsigned VPOpcode, |
0 |
| 525 |
bool hasFPExcept) { |
--- |
525 |
bool hasFPExcept) { |
--- |
| 526 |
// FIXME: Return strict opcodes in case of fp exceptions. |
--- |
526 |
// FIXME: Return strict opcodes in case of fp exceptions. |
--- |
| 527 |
switch (VPOpcode) { |
0 |
527 |
switch (VPOpcode) { |
0 |
| 528 |
default: |
0 |
528 |
default: |
0 |
| 529 |
return std::nullopt; |
0 |
529 |
return std::nullopt; |
0 |
| 530 |
#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC: |
--- |
530 |
#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC: |
--- |
| 531 |
#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC; |
--- |
531 |
#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC; |
--- |
| 532 |
#define END_REGISTER_VP_SDNODE(VPOPC) break; |
--- |
532 |
#define END_REGISTER_VP_SDNODE(VPOPC) break; |
--- |
| 533 |
#include "llvm/IR/VPIntrinsics.def" |
--- |
533 |
#include "llvm/IR/VPIntrinsics.def" |
--- |
| 534 |
} |
--- |
534 |
} |
--- |
| 535 |
return std::nullopt; |
0 |
535 |
return std::nullopt; |
0 |
| 536 |
} |
--- |
536 |
} |
--- |
| 537 |
|
--- |
537 |
|
--- |
| 538 |
unsigned ISD::getVPForBaseOpcode(unsigned Opcode) { |
0 |
538 |
unsigned ISD::getVPForBaseOpcode(unsigned Opcode) { |
0 |
| 539 |
switch (Opcode) { |
0 |
539 |
switch (Opcode) { |
0 |
| 540 |
default: |
0 |
540 |
default: |
0 |
| 541 |
llvm_unreachable("can not translate this Opcode to VP."); |
0 |
541 |
llvm_unreachable("can not translate this Opcode to VP."); |
0 |
| 542 |
#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break; |
--- |
542 |
#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break; |
--- |
| 543 |
#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC: |
--- |
543 |
#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC: |
--- |
| 544 |
#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC; |
--- |
544 |
#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC; |
--- |
| 545 |
#include "llvm/IR/VPIntrinsics.def" |
--- |
545 |
#include "llvm/IR/VPIntrinsics.def" |
--- |
| 546 |
} |
--- |
546 |
} |
--- |
| 547 |
} |
--- |
547 |
} |
--- |
| 548 |
|
--- |
548 |
|
--- |
| 549 |
ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) { |
0 |
549 |
ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) { |
0 |
| 550 |
switch (ExtType) { |
0 |
550 |
switch (ExtType) { |
0 |
| 551 |
case ISD::EXTLOAD: |
0 |
551 |
case ISD::EXTLOAD: |
0 |
| 552 |
return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND; |
0 |
552 |
return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND; |
0 |
| 553 |
case ISD::SEXTLOAD: |
0 |
553 |
case ISD::SEXTLOAD: |
0 |
| 554 |
return ISD::SIGN_EXTEND; |
0 |
554 |
return ISD::SIGN_EXTEND; |
0 |
| 555 |
case ISD::ZEXTLOAD: |
0 |
555 |
case ISD::ZEXTLOAD: |
0 |
| 556 |
return ISD::ZERO_EXTEND; |
0 |
556 |
return ISD::ZERO_EXTEND; |
0 |
| 557 |
default: |
0 |
557 |
default: |
0 |
| 558 |
break; |
0 |
558 |
break; |
0 |
| 559 |
} |
--- |
559 |
} |
--- |
| 560 |
|
--- |
560 |
|
--- |
| 561 |
llvm_unreachable("Invalid LoadExtType"); |
0 |
561 |
llvm_unreachable("Invalid LoadExtType"); |
0 |
| 562 |
} |
--- |
562 |
} |
--- |
| 563 |
|
--- |
563 |
|
--- |
| 564 |
ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { |
3 |
564 |
ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { |
3 |
| 565 |
// To perform this operation, we just need to swap the L and G bits of the |
--- |
565 |
// To perform this operation, we just need to swap the L and G bits of the |
--- |
| 566 |
// operation. |
--- |
566 |
// operation. |
--- |
| 567 |
unsigned OldL = (Operation >> 2) & 1; |
3 |
567 |
unsigned OldL = (Operation >> 2) & 1; |
3 |
| 568 |
unsigned OldG = (Operation >> 1) & 1; |
3 |
568 |
unsigned OldG = (Operation >> 1) & 1; |
3 |
| 569 |
return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits |
3 |
569 |
return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits |
3 |
| 570 |
(OldL << 1) | // New G bit |
3 |
570 |
(OldL << 1) | // New G bit |
3 |
| 571 |
(OldG << 2)); // New L bit. |
3 |
571 |
(OldG << 2)); // New L bit. |
3 |
| 572 |
} |
--- |
572 |
} |
--- |
| 573 |
|
--- |
573 |
|
--- |
| 574 |
static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) { |
1 |
574 |
static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) { |
1 |
| 575 |
unsigned Operation = Op; |
1 |
575 |
unsigned Operation = Op; |
1 |
| 576 |
if (isIntegerLike) |
1 |
576 |
if (isIntegerLike) |
1 |
| 577 |
Operation ^= 7; // Flip L, G, E bits, but not U. |
1 |
577 |
Operation ^= 7; // Flip L, G, E bits, but not U. |
1 |
| 578 |
else |
--- |
578 |
else |
--- |
| 579 |
Operation ^= 15; // Flip all of the condition bits. |
0 |
579 |
Operation ^= 15; // Flip all of the condition bits. |
0 |
| 580 |
|
--- |
580 |
|
--- |
| 581 |
if (Operation > ISD::SETTRUE2) |
1 |
581 |
if (Operation > ISD::SETTRUE2) |
1 |
| 582 |
Operation &= ~8; // Don't let N and U bits get set. |
0 |
582 |
Operation &= ~8; // Don't let N and U bits get set. |
0 |
| 583 |
|
--- |
583 |
|
--- |
| 584 |
return ISD::CondCode(Operation); |
1 |
584 |
return ISD::CondCode(Operation); |
1 |
| 585 |
} |
--- |
585 |
} |
--- |
| 586 |
|
--- |
586 |
|
--- |
| 587 |
ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) { |
1 |
587 |
ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) { |
1 |
| 588 |
return getSetCCInverseImpl(Op, Type.isInteger()); |
1 |
588 |
return getSetCCInverseImpl(Op, Type.isInteger()); |
1 |
| 589 |
} |
--- |
589 |
} |
--- |
| 590 |
|
--- |
590 |
|
--- |
| 591 |
ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op, |
0 |
591 |
ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op, |
0 |
| 592 |
bool isIntegerLike) { |
--- |
592 |
bool isIntegerLike) { |
--- |
| 593 |
return getSetCCInverseImpl(Op, isIntegerLike); |
0 |
593 |
return getSetCCInverseImpl(Op, isIntegerLike); |
0 |
| 594 |
} |
--- |
594 |
} |
--- |
| 595 |
|
--- |
595 |
|
--- |
| 596 |
/// For an integer comparison, return 1 if the comparison is a signed operation |
--- |
596 |
/// For an integer comparison, return 1 if the comparison is a signed operation |
--- |
| 597 |
/// and 2 if the result is an unsigned comparison. Return zero if the operation |
--- |
597 |
/// and 2 if the result is an unsigned comparison. Return zero if the operation |
--- |
| 598 |
/// does not depend on the sign of the input (setne and seteq). |
--- |
598 |
/// does not depend on the sign of the input (setne and seteq). |
--- |
| 599 |
static int isSignedOp(ISD::CondCode Opcode) { |
0 |
599 |
static int isSignedOp(ISD::CondCode Opcode) { |
0 |
| 600 |
switch (Opcode) { |
0 |
600 |
switch (Opcode) { |
0 |
| 601 |
default: llvm_unreachable("Illegal integer setcc operation!"); |
0 |
601 |
default: llvm_unreachable("Illegal integer setcc operation!"); |
0 |
| 602 |
case ISD::SETEQ: |
0 |
602 |
case ISD::SETEQ: |
0 |
| 603 |
case ISD::SETNE: return 0; |
0 |
603 |
case ISD::SETNE: return 0; |
0 |
| 604 |
case ISD::SETLT: |
0 |
604 |
case ISD::SETLT: |
0 |
| 605 |
case ISD::SETLE: |
--- |
605 |
case ISD::SETLE: |
--- |
| 606 |
case ISD::SETGT: |
--- |
606 |
case ISD::SETGT: |
--- |
| 607 |
case ISD::SETGE: return 1; |
0 |
607 |
case ISD::SETGE: return 1; |
0 |
| 608 |
case ISD::SETULT: |
0 |
608 |
case ISD::SETULT: |
0 |
| 609 |
case ISD::SETULE: |
--- |
609 |
case ISD::SETULE: |
--- |
| 610 |
case ISD::SETUGT: |
--- |
610 |
case ISD::SETUGT: |
--- |
| 611 |
case ISD::SETUGE: return 2; |
0 |
611 |
case ISD::SETUGE: return 2; |
0 |
| 612 |
} |
--- |
612 |
} |
--- |
| 613 |
} |
--- |
613 |
} |
--- |
| 614 |
|
--- |
614 |
|
--- |
| 615 |
ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, |
0 |
615 |
ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, |
0 |
| 616 |
EVT Type) { |
--- |
616 |
EVT Type) { |
--- |
| 617 |
bool IsInteger = Type.isInteger(); |
0 |
617 |
bool IsInteger = Type.isInteger(); |
0 |
| 618 |
if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) |
0 |
618 |
if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) |
0 |
| 619 |
// Cannot fold a signed integer setcc with an unsigned integer setcc. |
--- |
619 |
// Cannot fold a signed integer setcc with an unsigned integer setcc. |
--- |
| 620 |
return ISD::SETCC_INVALID; |
0 |
620 |
return ISD::SETCC_INVALID; |
0 |
| 621 |
|
--- |
621 |
|
--- |
| 622 |
unsigned Op = Op1 | Op2; // Combine all of the condition bits. |
0 |
622 |
unsigned Op = Op1 | Op2; // Combine all of the condition bits. |
0 |
| 623 |
|
--- |
623 |
|
--- |
| 624 |
// If the N and U bits get set, then the resultant comparison DOES suddenly |
--- |
624 |
// If the N and U bits get set, then the resultant comparison DOES suddenly |
--- |
| 625 |
// care about orderedness, and it is true when ordered. |
--- |
625 |
// care about orderedness, and it is true when ordered. |
--- |
| 626 |
if (Op > ISD::SETTRUE2) |
0 |
626 |
if (Op > ISD::SETTRUE2) |
0 |
| 627 |
Op &= ~16; // Clear the U bit if the N bit is set. |
0 |
627 |
Op &= ~16; // Clear the U bit if the N bit is set. |
0 |
| 628 |
|
--- |
628 |
|
--- |
| 629 |
// Canonicalize illegal integer setcc's. |
--- |
629 |
// Canonicalize illegal integer setcc's. |
--- |
| 630 |
if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT |
0 |
630 |
if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT |
0 |
| 631 |
Op = ISD::SETNE; |
0 |
631 |
Op = ISD::SETNE; |
0 |
| 632 |
|
--- |
632 |
|
--- |
| 633 |
return ISD::CondCode(Op); |
0 |
633 |
return ISD::CondCode(Op); |
0 |
| 634 |
} |
--- |
634 |
} |
--- |
| 635 |
|
--- |
635 |
|
--- |
| 636 |
ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, |
0 |
636 |
ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, |
0 |
| 637 |
EVT Type) { |
--- |
637 |
EVT Type) { |
--- |
| 638 |
bool IsInteger = Type.isInteger(); |
0 |
638 |
bool IsInteger = Type.isInteger(); |
0 |
| 639 |
if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) |
0 |
639 |
if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) |
0 |
| 640 |
// Cannot fold a signed setcc with an unsigned setcc. |
--- |
640 |
// Cannot fold a signed setcc with an unsigned setcc. |
--- |
| 641 |
return ISD::SETCC_INVALID; |
0 |
641 |
return ISD::SETCC_INVALID; |
0 |
| 642 |
|
--- |
642 |
|
--- |
| 643 |
// Combine all of the condition bits. |
--- |
643 |
// Combine all of the condition bits. |
--- |
| 644 |
ISD::CondCode Result = ISD::CondCode(Op1 & Op2); |
0 |
644 |
ISD::CondCode Result = ISD::CondCode(Op1 & Op2); |
0 |
| 645 |
|
--- |
645 |
|
--- |
| 646 |
// Canonicalize illegal integer setcc's. |
--- |
646 |
// Canonicalize illegal integer setcc's. |
--- |
| 647 |
if (IsInteger) { |
0 |
647 |
if (IsInteger) { |
0 |
| 648 |
switch (Result) { |
0 |
648 |
switch (Result) { |
0 |
| 649 |
default: break; |
0 |
649 |
default: break; |
0 |
| 650 |
case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT |
0 |
650 |
case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT |
0 |
| 651 |
case ISD::SETOEQ: // SETEQ & SETU[LG]E |
0 |
651 |
case ISD::SETOEQ: // SETEQ & SETU[LG]E |
0 |
| 652 |
case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE |
0 |
652 |
case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE |
0 |
| 653 |
case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE |
0 |
653 |
case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE |
0 |
| 654 |
case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE |
0 |
654 |
case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE |
0 |
| 655 |
} |
--- |
655 |
} |
--- |
| 656 |
} |
--- |
656 |
} |
--- |
| 657 |
|
--- |
657 |
|
--- |
| 658 |
return Result; |
0 |
658 |
return Result; |
0 |
| 659 |
} |
--- |
659 |
} |
--- |
| 660 |
|
--- |
660 |
|
--- |
| 661 |
//===----------------------------------------------------------------------===// |
--- |
661 |
//===----------------------------------------------------------------------===// |
--- |
| 662 |
// SDNode Profile Support |
--- |
662 |
// SDNode Profile Support |
--- |
| 663 |
//===----------------------------------------------------------------------===// |
--- |
663 |
//===----------------------------------------------------------------------===// |
--- |
| 664 |
|
--- |
664 |
|
--- |
| 665 |
/// AddNodeIDOpcode - Add the node opcode to the NodeID data. |
--- |
665 |
/// AddNodeIDOpcode - Add the node opcode to the NodeID data. |
--- |
| 666 |
static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { |
132 |
666 |
static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { |
145 |
| 667 |
ID.AddInteger(OpC); |
132 |
667 |
ID.AddInteger(OpC); |
145 |
| 668 |
} |
132 |
668 |
} |
145 |
| 669 |
|
--- |
669 |
|
--- |
| 670 |
/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them |
--- |
670 |
/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them |
--- |
| 671 |
/// solely with their pointer. |
--- |
671 |
/// solely with their pointer. |
--- |
| 672 |
static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { |
132 |
672 |
static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { |
145 |
| 673 |
ID.AddPointer(VTList.VTs); |
132 |
673 |
ID.AddPointer(VTList.VTs); |
145 |
| 674 |
} |
132 |
674 |
} |
145 |
| 675 |
|
--- |
675 |
|
--- |
| 676 |
/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. |
--- |
676 |
/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. |
--- |
| 677 |
static void AddNodeIDOperands(FoldingSetNodeID &ID, |
96 |
677 |
static void AddNodeIDOperands(FoldingSetNodeID &ID, |
96 |
| 678 |
ArrayRef Ops) { |
--- |
678 |
ArrayRef Ops) { |
--- |
| 679 |
for (const auto &Op : Ops) { |
215 |
679 |
for (const auto &Op : Ops) { |
215 |
| 680 |
ID.AddPointer(Op.getNode()); |
119 |
680 |
ID.AddPointer(Op.getNode()); |
119 |
| 681 |
ID.AddInteger(Op.getResNo()); |
119 |
681 |
ID.AddInteger(Op.getResNo()); |
119 |
| 682 |
} |
--- |
682 |
} |
--- |
| 683 |
} |
96 |
683 |
} |
96 |
| 684 |
|
--- |
684 |
|
--- |
| 685 |
/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. |
--- |
685 |
/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. |
--- |
| 686 |
static void AddNodeIDOperands(FoldingSetNodeID &ID, |
36 |
686 |
static void AddNodeIDOperands(FoldingSetNodeID &ID, |
49 |
| 687 |
ArrayRef Ops) { |
--- |
687 |
ArrayRef Ops) { |
--- |
| 688 |
for (const auto &Op : Ops) { |
67 |
688 |
for (const auto &Op : Ops) { |
72 |
| 689 |
ID.AddPointer(Op.getNode()); |
31 |
689 |
ID.AddPointer(Op.getNode()); |
23 |
| 690 |
ID.AddInteger(Op.getResNo()); |
31 |
690 |
ID.AddInteger(Op.getResNo()); |
23 |
| 691 |
} |
--- |
691 |
} |
--- |
| 692 |
} |
36 |
692 |
} |
49 |
| 693 |
|
--- |
693 |
|
--- |
| 694 |
static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC, |
96 |
694 |
static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC, |
96 |
| 695 |
SDVTList VTList, ArrayRef OpList) { |
--- |
695 |
SDVTList VTList, ArrayRef OpList) { |
--- |
| 696 |
AddNodeIDOpcode(ID, OpC); |
96 |
696 |
AddNodeIDOpcode(ID, OpC); |
96 |
| 697 |
AddNodeIDValueTypes(ID, VTList); |
96 |
697 |
AddNodeIDValueTypes(ID, VTList); |
96 |
| 698 |
AddNodeIDOperands(ID, OpList); |
96 |
698 |
AddNodeIDOperands(ID, OpList); |
96 |
| 699 |
} |
96 |
699 |
} |
96 |
| 700 |
|
--- |
700 |
|
--- |
| 701 |
/// If this is an SDNode with special info, add this info to the NodeID data. |
--- |
701 |
/// If this is an SDNode with special info, add this info to the NodeID data. |
--- |
| 702 |
static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { |
37 |
702 |
static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { |
50 |
| 703 |
switch (N->getOpcode()) { |
37 |
703 |
switch (N->getOpcode()) { |
50 |
| 704 |
case ISD::TargetExternalSymbol: |
0 |
704 |
case ISD::TargetExternalSymbol: |
0 |
| 705 |
case ISD::ExternalSymbol: |
--- |
705 |
case ISD::ExternalSymbol: |
--- |
| 706 |
case ISD::MCSymbol: |
--- |
706 |
case ISD::MCSymbol: |
--- |
| 707 |
llvm_unreachable("Should only be used on nodes with operands"); |
0 |
707 |
llvm_unreachable("Should only be used on nodes with operands"); |
0 |
| 708 |
default: break; // Normal nodes don't need extra info. |
15 |
708 |
default: break; // Normal nodes don't need extra info. |
22 |
| 709 |
case ISD::TargetConstant: |
10 |
709 |
case ISD::TargetConstant: |
15 |
| 710 |
case ISD::Constant: { |
--- |
710 |
case ISD::Constant: { |
--- |
| 711 |
const ConstantSDNode *C = cast(N); |
10 |
711 |
const ConstantSDNode *C = cast(N); |
15 |
| 712 |
ID.AddPointer(C->getConstantIntValue()); |
10 |
712 |
ID.AddPointer(C->getConstantIntValue()); |
15 |
| 713 |
ID.AddBoolean(C->isOpaque()); |
10 |
713 |
ID.AddBoolean(C->isOpaque()); |
15 |
| 714 |
break; |
10 |
714 |
break; |
15 |
| 715 |
} |
--- |
715 |
} |
--- |
| 716 |
case ISD::TargetConstantFP: |
0 |
716 |
case ISD::TargetConstantFP: |
0 |
| 717 |
case ISD::ConstantFP: |
--- |
717 |
case ISD::ConstantFP: |
--- |
| 718 |
ID.AddPointer(cast(N)->getConstantFPValue()); |
0 |
718 |
ID.AddPointer(cast(N)->getConstantFPValue()); |
0 |
| 719 |
break; |
0 |
719 |
break; |
0 |
| 720 |
case ISD::TargetGlobalAddress: |
0 |
720 |
case ISD::TargetGlobalAddress: |
0 |
| 721 |
case ISD::GlobalAddress: |
--- |
721 |
case ISD::GlobalAddress: |
--- |
| 722 |
case ISD::TargetGlobalTLSAddress: |
--- |
722 |
case ISD::TargetGlobalTLSAddress: |
--- |
| 723 |
case ISD::GlobalTLSAddress: { |
--- |
723 |
case ISD::GlobalTLSAddress: { |
--- |
| 724 |
const GlobalAddressSDNode *GA = cast(N); |
0 |
724 |
const GlobalAddressSDNode *GA = cast(N); |
0 |
| 725 |
ID.AddPointer(GA->getGlobal()); |
0 |
725 |
ID.AddPointer(GA->getGlobal()); |
0 |
| 726 |
ID.AddInteger(GA->getOffset()); |
0 |
726 |
ID.AddInteger(GA->getOffset()); |
0 |
| 727 |
ID.AddInteger(GA->getTargetFlags()); |
0 |
727 |
ID.AddInteger(GA->getTargetFlags()); |
0 |
| 728 |
break; |
0 |
728 |
break; |
0 |
| 729 |
} |
--- |
729 |
} |
--- |
| 730 |
case ISD::BasicBlock: |
0 |
730 |
case ISD::BasicBlock: |
1 |
| 731 |
ID.AddPointer(cast(N)->getBasicBlock()); |
0 |
731 |
ID.AddPointer(cast(N)->getBasicBlock()); |
1 |
| 732 |
break; |
0 |
732 |
break; |
1 |
| 733 |
case ISD::Register: |
2 |
733 |
case ISD::Register: |
4 |
| 734 |
ID.AddInteger(cast(N)->getReg()); |
2 |
734 |
ID.AddInteger(cast(N)->getReg()); |
4 |
| 735 |
break; |
2 |
735 |
break; |
4 |
| 736 |
case ISD::RegisterMask: |
0 |
736 |
case ISD::RegisterMask: |
0 |
| 737 |
ID.AddPointer(cast(N)->getRegMask()); |
0 |
737 |
ID.AddPointer(cast(N)->getRegMask()); |
0 |
| 738 |
break; |
0 |
738 |
break; |
0 |
| 739 |
case ISD::SRCVALUE: |
0 |
739 |
case ISD::SRCVALUE: |
0 |
| 740 |
ID.AddPointer(cast(N)->getValue()); |
0 |
740 |
ID.AddPointer(cast(N)->getValue()); |
0 |
| 741 |
break; |
0 |
741 |
break; |
0 |
| 742 |
case ISD::FrameIndex: |
7 |
742 |
case ISD::FrameIndex: |
7 |
| 743 |
case ISD::TargetFrameIndex: |
--- |
743 |
case ISD::TargetFrameIndex: |
--- |
| 744 |
ID.AddInteger(cast(N)->getIndex()); |
7 |
744 |
ID.AddInteger(cast(N)->getIndex()); |
7 |
| 745 |
break; |
7 |
745 |
break; |
7 |
| 746 |
case ISD::LIFETIME_START: |
0 |
746 |
case ISD::LIFETIME_START: |
0 |
| 747 |
case ISD::LIFETIME_END: |
--- |
747 |
case ISD::LIFETIME_END: |
--- |
| 748 |
if (cast(N)->hasOffset()) { |
0 |
748 |
if (cast(N)->hasOffset()) { |
0 |
| 749 |
ID.AddInteger(cast(N)->getSize()); |
0 |
749 |
ID.AddInteger(cast(N)->getSize()); |
0 |
| 750 |
ID.AddInteger(cast(N)->getOffset()); |
0 |
750 |
ID.AddInteger(cast(N)->getOffset()); |
0 |
| 751 |
} |
--- |
751 |
} |
--- |
| 752 |
break; |
0 |
752 |
break; |
0 |
| 753 |
case ISD::PSEUDO_PROBE: |
0 |
753 |
case ISD::PSEUDO_PROBE: |
0 |
| 754 |
ID.AddInteger(cast(N)->getGuid()); |
0 |
754 |
ID.AddInteger(cast(N)->getGuid()); |
0 |
| 755 |
ID.AddInteger(cast(N)->getIndex()); |
0 |
755 |
ID.AddInteger(cast(N)->getIndex()); |
0 |
| 756 |
ID.AddInteger(cast(N)->getAttributes()); |
0 |
756 |
ID.AddInteger(cast(N)->getAttributes()); |
0 |
| 757 |
break; |
0 |
757 |
break; |
0 |
| 758 |
case ISD::JumpTable: |
0 |
758 |
case ISD::JumpTable: |
0 |
| 759 |
case ISD::TargetJumpTable: |
--- |
759 |
case ISD::TargetJumpTable: |
--- |
| 760 |
ID.AddInteger(cast(N)->getIndex()); |
0 |
760 |
ID.AddInteger(cast(N)->getIndex()); |
0 |
| 761 |
ID.AddInteger(cast(N)->getTargetFlags()); |
0 |
761 |
ID.AddInteger(cast(N)->getTargetFlags()); |
0 |
| 762 |
break; |
0 |
762 |
break; |
0 |
| 763 |
case ISD::ConstantPool: |
0 |
763 |
case ISD::ConstantPool: |
0 |
| 764 |
case ISD::TargetConstantPool: { |
--- |
764 |
case ISD::TargetConstantPool: { |
--- |
| 765 |
const ConstantPoolSDNode *CP = cast(N); |
0 |
765 |
const ConstantPoolSDNode *CP = cast(N); |
0 |
| 766 |
ID.AddInteger(CP->getAlign().value()); |
0 |
766 |
ID.AddInteger(CP->getAlign().value()); |
0 |
| 767 |
ID.AddInteger(CP->getOffset()); |
0 |
767 |
ID.AddInteger(CP->getOffset()); |
0 |
| 768 |
if (CP->isMachineConstantPoolEntry()) |
0 |
768 |
if (CP->isMachineConstantPoolEntry()) |
0 |
| 769 |
CP->getMachineCPVal()->addSelectionDAGCSEId(ID); |
0 |
769 |
CP->getMachineCPVal()->addSelectionDAGCSEId(ID); |
0 |
| 770 |
else |
--- |
770 |
else |
--- |
| 771 |
ID.AddPointer(CP->getConstVal()); |
0 |
771 |
ID.AddPointer(CP->getConstVal()); |
0 |
| 772 |
ID.AddInteger(CP->getTargetFlags()); |
0 |
772 |
ID.AddInteger(CP->getTargetFlags()); |
0 |
| 773 |
break; |
0 |
773 |
break; |
0 |
| 774 |
} |
--- |
774 |
} |
--- |
| 775 |
case ISD::TargetIndex: { |
0 |
775 |
case ISD::TargetIndex: { |
0 |
| 776 |
const TargetIndexSDNode *TI = cast(N); |
0 |
776 |
const TargetIndexSDNode *TI = cast(N); |
0 |
| 777 |
ID.AddInteger(TI->getIndex()); |
0 |
777 |
ID.AddInteger(TI->getIndex()); |
0 |
| 778 |
ID.AddInteger(TI->getOffset()); |
0 |
778 |
ID.AddInteger(TI->getOffset()); |
0 |
| 779 |
ID.AddInteger(TI->getTargetFlags()); |
0 |
779 |
ID.AddInteger(TI->getTargetFlags()); |
0 |
| 780 |
break; |
0 |
780 |
break; |
0 |
| 781 |
} |
--- |
781 |
} |
--- |
| 782 |
case ISD::LOAD: { |
2 |
782 |
case ISD::LOAD: { |
0 |
| 783 |
const LoadSDNode *LD = cast(N); |
2 |
783 |
const LoadSDNode *LD = cast(N); |
0 |
| 784 |
ID.AddInteger(LD->getMemoryVT().getRawBits()); |
2 |
784 |
ID.AddInteger(LD->getMemoryVT().getRawBits()); |
0 |
| 785 |
ID.AddInteger(LD->getRawSubclassData()); |
2 |
785 |
ID.AddInteger(LD->getRawSubclassData()); |
0 |
| 786 |
ID.AddInteger(LD->getPointerInfo().getAddrSpace()); |
2 |
786 |
ID.AddInteger(LD->getPointerInfo().getAddrSpace()); |
0 |
| 787 |
ID.AddInteger(LD->getMemOperand()->getFlags()); |
2 |
787 |
ID.AddInteger(LD->getMemOperand()->getFlags()); |
0 |
| 788 |
break; |
2 |
788 |
break; |
0 |
| 789 |
} |
--- |
789 |
} |
--- |
| 790 |
case ISD::STORE: { |
1 |
790 |
case ISD::STORE: { |
1 |
| 791 |
const StoreSDNode *ST = cast(N); |
1 |
791 |
const StoreSDNode *ST = cast(N); |
1 |
| 792 |
ID.AddInteger(ST->getMemoryVT().getRawBits()); |
1 |
792 |
ID.AddInteger(ST->getMemoryVT().getRawBits()); |
1 |
| 793 |
ID.AddInteger(ST->getRawSubclassData()); |
1 |
793 |
ID.AddInteger(ST->getRawSubclassData()); |
1 |
| 794 |
ID.AddInteger(ST->getPointerInfo().getAddrSpace()); |
1 |
794 |
ID.AddInteger(ST->getPointerInfo().getAddrSpace()); |
1 |
| 795 |
ID.AddInteger(ST->getMemOperand()->getFlags()); |
1 |
795 |
ID.AddInteger(ST->getMemOperand()->getFlags()); |
1 |
| 796 |
break; |
1 |
796 |
break; |
1 |
| 797 |
} |
--- |
797 |
} |
--- |
| 798 |
case ISD::VP_LOAD: { |
0 |
798 |
case ISD::VP_LOAD: { |
0 |
| 799 |
const VPLoadSDNode *ELD = cast(N); |
0 |
799 |
const VPLoadSDNode *ELD = cast(N); |
0 |
| 800 |
ID.AddInteger(ELD->getMemoryVT().getRawBits()); |
0 |
800 |
ID.AddInteger(ELD->getMemoryVT().getRawBits()); |
0 |
| 801 |
ID.AddInteger(ELD->getRawSubclassData()); |
0 |
801 |
ID.AddInteger(ELD->getRawSubclassData()); |
0 |
| 802 |
ID.AddInteger(ELD->getPointerInfo().getAddrSpace()); |
0 |
802 |
ID.AddInteger(ELD->getPointerInfo().getAddrSpace()); |
0 |
| 803 |
ID.AddInteger(ELD->getMemOperand()->getFlags()); |
0 |
803 |
ID.AddInteger(ELD->getMemOperand()->getFlags()); |
0 |
| 804 |
break; |
0 |
804 |
break; |
0 |
| 805 |
} |
--- |
805 |
} |
--- |
| 806 |
case ISD::VP_STORE: { |
0 |
806 |
case ISD::VP_STORE: { |
0 |
| 807 |
const VPStoreSDNode *EST = cast(N); |
0 |
807 |
const VPStoreSDNode *EST = cast(N); |
0 |
| 808 |
ID.AddInteger(EST->getMemoryVT().getRawBits()); |
0 |
808 |
ID.AddInteger(EST->getMemoryVT().getRawBits()); |
0 |
| 809 |
ID.AddInteger(EST->getRawSubclassData()); |
0 |
809 |
ID.AddInteger(EST->getRawSubclassData()); |
0 |
| 810 |
ID.AddInteger(EST->getPointerInfo().getAddrSpace()); |
0 |
810 |
ID.AddInteger(EST->getPointerInfo().getAddrSpace()); |
0 |
| 811 |
ID.AddInteger(EST->getMemOperand()->getFlags()); |
0 |
811 |
ID.AddInteger(EST->getMemOperand()->getFlags()); |
0 |
| 812 |
break; |
0 |
812 |
break; |
0 |
| 813 |
} |
--- |
813 |
} |
--- |
| 814 |
case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: { |
0 |
814 |
case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: { |
0 |
| 815 |
const VPStridedLoadSDNode *SLD = cast(N); |
0 |
815 |
const VPStridedLoadSDNode *SLD = cast(N); |
0 |
| 816 |
ID.AddInteger(SLD->getMemoryVT().getRawBits()); |
0 |
816 |
ID.AddInteger(SLD->getMemoryVT().getRawBits()); |
0 |
| 817 |
ID.AddInteger(SLD->getRawSubclassData()); |
0 |
817 |
ID.AddInteger(SLD->getRawSubclassData()); |
0 |
| 818 |
ID.AddInteger(SLD->getPointerInfo().getAddrSpace()); |
0 |
818 |
ID.AddInteger(SLD->getPointerInfo().getAddrSpace()); |
0 |
| 819 |
break; |
0 |
819 |
break; |
0 |
| 820 |
} |
--- |
820 |
} |
--- |
| 821 |
case ISD::EXPERIMENTAL_VP_STRIDED_STORE: { |
0 |
821 |
case ISD::EXPERIMENTAL_VP_STRIDED_STORE: { |
0 |
| 822 |
const VPStridedStoreSDNode *SST = cast(N); |
0 |
822 |
const VPStridedStoreSDNode *SST = cast(N); |
0 |
| 823 |
ID.AddInteger(SST->getMemoryVT().getRawBits()); |
0 |
823 |
ID.AddInteger(SST->getMemoryVT().getRawBits()); |
0 |
| 824 |
ID.AddInteger(SST->getRawSubclassData()); |
0 |
824 |
ID.AddInteger(SST->getRawSubclassData()); |
0 |
| 825 |
ID.AddInteger(SST->getPointerInfo().getAddrSpace()); |
0 |
825 |
ID.AddInteger(SST->getPointerInfo().getAddrSpace()); |
0 |
| 826 |
break; |
0 |
826 |
break; |
0 |
| 827 |
} |
--- |
827 |
} |
--- |
| 828 |
case ISD::VP_GATHER: { |
0 |
828 |
case ISD::VP_GATHER: { |
0 |
| 829 |
const VPGatherSDNode *EG = cast(N); |
0 |
829 |
const VPGatherSDNode *EG = cast(N); |
0 |
| 830 |
ID.AddInteger(EG->getMemoryVT().getRawBits()); |
0 |
830 |
ID.AddInteger(EG->getMemoryVT().getRawBits()); |
0 |
| 831 |
ID.AddInteger(EG->getRawSubclassData()); |
0 |
831 |
ID.AddInteger(EG->getRawSubclassData()); |
0 |
| 832 |
ID.AddInteger(EG->getPointerInfo().getAddrSpace()); |
0 |
832 |
ID.AddInteger(EG->getPointerInfo().getAddrSpace()); |
0 |
| 833 |
ID.AddInteger(EG->getMemOperand()->getFlags()); |
0 |
833 |
ID.AddInteger(EG->getMemOperand()->getFlags()); |
0 |
| 834 |
break; |
0 |
834 |
break; |
0 |
| 835 |
} |
--- |
835 |
} |
--- |
| 836 |
case ISD::VP_SCATTER: { |
0 |
836 |
case ISD::VP_SCATTER: { |
0 |
| 837 |
const VPScatterSDNode *ES = cast(N); |
0 |
837 |
const VPScatterSDNode *ES = cast(N); |
0 |
| 838 |
ID.AddInteger(ES->getMemoryVT().getRawBits()); |
0 |
838 |
ID.AddInteger(ES->getMemoryVT().getRawBits()); |
0 |
| 839 |
ID.AddInteger(ES->getRawSubclassData()); |
0 |
839 |
ID.AddInteger(ES->getRawSubclassData()); |
0 |
| 840 |
ID.AddInteger(ES->getPointerInfo().getAddrSpace()); |
0 |
840 |
ID.AddInteger(ES->getPointerInfo().getAddrSpace()); |
0 |
| 841 |
ID.AddInteger(ES->getMemOperand()->getFlags()); |
0 |
841 |
ID.AddInteger(ES->getMemOperand()->getFlags()); |
0 |
| 842 |
break; |
0 |
842 |
break; |
0 |
| 843 |
} |
--- |
843 |
} |
--- |
| 844 |
case ISD::MLOAD: { |
0 |
844 |
case ISD::MLOAD: { |
0 |
| 845 |
const MaskedLoadSDNode *MLD = cast(N); |
0 |
845 |
const MaskedLoadSDNode *MLD = cast(N); |
0 |
| 846 |
ID.AddInteger(MLD->getMemoryVT().getRawBits()); |
0 |
846 |
ID.AddInteger(MLD->getMemoryVT().getRawBits()); |
0 |
| 847 |
ID.AddInteger(MLD->getRawSubclassData()); |
0 |
847 |
ID.AddInteger(MLD->getRawSubclassData()); |
0 |
| 848 |
ID.AddInteger(MLD->getPointerInfo().getAddrSpace()); |
0 |
848 |
ID.AddInteger(MLD->getPointerInfo().getAddrSpace()); |
0 |
| 849 |
ID.AddInteger(MLD->getMemOperand()->getFlags()); |
0 |
849 |
ID.AddInteger(MLD->getMemOperand()->getFlags()); |
0 |
| 850 |
break; |
0 |
850 |
break; |
0 |
| 851 |
} |
--- |
851 |
} |
--- |
| 852 |
case ISD::MSTORE: { |
0 |
852 |
case ISD::MSTORE: { |
0 |
| 853 |
const MaskedStoreSDNode *MST = cast(N); |
0 |
853 |
const MaskedStoreSDNode *MST = cast(N); |
0 |
| 854 |
ID.AddInteger(MST->getMemoryVT().getRawBits()); |
0 |
854 |
ID.AddInteger(MST->getMemoryVT().getRawBits()); |
0 |
| 855 |
ID.AddInteger(MST->getRawSubclassData()); |
0 |
855 |
ID.AddInteger(MST->getRawSubclassData()); |
0 |
| 856 |
ID.AddInteger(MST->getPointerInfo().getAddrSpace()); |
0 |
856 |
ID.AddInteger(MST->getPointerInfo().getAddrSpace()); |
0 |
| 857 |
ID.AddInteger(MST->getMemOperand()->getFlags()); |
0 |
857 |
ID.AddInteger(MST->getMemOperand()->getFlags()); |
0 |
| 858 |
break; |
0 |
858 |
break; |
0 |
| 859 |
} |
--- |
859 |
} |
--- |
| 860 |
case ISD::MGATHER: { |
0 |
860 |
case ISD::MGATHER: { |
0 |
| 861 |
const MaskedGatherSDNode *MG = cast(N); |
0 |
861 |
const MaskedGatherSDNode *MG = cast(N); |
0 |
| 862 |
ID.AddInteger(MG->getMemoryVT().getRawBits()); |
0 |
862 |
ID.AddInteger(MG->getMemoryVT().getRawBits()); |
0 |
| 863 |
ID.AddInteger(MG->getRawSubclassData()); |
0 |
863 |
ID.AddInteger(MG->getRawSubclassData()); |
0 |
| 864 |
ID.AddInteger(MG->getPointerInfo().getAddrSpace()); |
0 |
864 |
ID.AddInteger(MG->getPointerInfo().getAddrSpace()); |
0 |
| 865 |
ID.AddInteger(MG->getMemOperand()->getFlags()); |
0 |
865 |
ID.AddInteger(MG->getMemOperand()->getFlags()); |
0 |
| 866 |
break; |
0 |
866 |
break; |
0 |
| 867 |
} |
--- |
867 |
} |
--- |
| 868 |
case ISD::MSCATTER: { |
0 |
868 |
case ISD::MSCATTER: { |
0 |
| 869 |
const MaskedScatterSDNode *MS = cast(N); |
0 |
869 |
const MaskedScatterSDNode *MS = cast(N); |
0 |
| 870 |
ID.AddInteger(MS->getMemoryVT().getRawBits()); |
0 |
870 |
ID.AddInteger(MS->getMemoryVT().getRawBits()); |
0 |
| 871 |
ID.AddInteger(MS->getRawSubclassData()); |
0 |
871 |
ID.AddInteger(MS->getRawSubclassData()); |
0 |
| 872 |
ID.AddInteger(MS->getPointerInfo().getAddrSpace()); |
0 |
872 |
ID.AddInteger(MS->getPointerInfo().getAddrSpace()); |
0 |
| 873 |
ID.AddInteger(MS->getMemOperand()->getFlags()); |
0 |
873 |
ID.AddInteger(MS->getMemOperand()->getFlags()); |
0 |
| 874 |
break; |
0 |
874 |
break; |
0 |
| 875 |
} |
--- |
875 |
} |
--- |
| 876 |
case ISD::ATOMIC_CMP_SWAP: |
0 |
876 |
case ISD::ATOMIC_CMP_SWAP: |
0 |
| 877 |
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: |
--- |
877 |
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: |
--- |
| 878 |
case ISD::ATOMIC_SWAP: |
--- |
878 |
case ISD::ATOMIC_SWAP: |
--- |
| 879 |
case ISD::ATOMIC_LOAD_ADD: |
--- |
879 |
case ISD::ATOMIC_LOAD_ADD: |
--- |
| 880 |
case ISD::ATOMIC_LOAD_SUB: |
--- |
880 |
case ISD::ATOMIC_LOAD_SUB: |
--- |
| 881 |
case ISD::ATOMIC_LOAD_AND: |
--- |
881 |
case ISD::ATOMIC_LOAD_AND: |
--- |
| 882 |
case ISD::ATOMIC_LOAD_CLR: |
--- |
882 |
case ISD::ATOMIC_LOAD_CLR: |
--- |
| 883 |
case ISD::ATOMIC_LOAD_OR: |
--- |
883 |
case ISD::ATOMIC_LOAD_OR: |
--- |
| 884 |
case ISD::ATOMIC_LOAD_XOR: |
--- |
884 |
case ISD::ATOMIC_LOAD_XOR: |
--- |
| 885 |
case ISD::ATOMIC_LOAD_NAND: |
--- |
885 |
case ISD::ATOMIC_LOAD_NAND: |
--- |
| 886 |
case ISD::ATOMIC_LOAD_MIN: |
--- |
886 |
case ISD::ATOMIC_LOAD_MIN: |
--- |
| 887 |
case ISD::ATOMIC_LOAD_MAX: |
--- |
887 |
case ISD::ATOMIC_LOAD_MAX: |
--- |
| 888 |
case ISD::ATOMIC_LOAD_UMIN: |
--- |
888 |
case ISD::ATOMIC_LOAD_UMIN: |
--- |
| 889 |
case ISD::ATOMIC_LOAD_UMAX: |
--- |
889 |
case ISD::ATOMIC_LOAD_UMAX: |
--- |
| 890 |
case ISD::ATOMIC_LOAD: |
--- |
890 |
case ISD::ATOMIC_LOAD: |
--- |
| 891 |
case ISD::ATOMIC_STORE: { |
--- |
891 |
case ISD::ATOMIC_STORE: { |
--- |
| 892 |
const AtomicSDNode *AT = cast(N); |
0 |
892 |
const AtomicSDNode *AT = cast(N); |
0 |
| 893 |
ID.AddInteger(AT->getMemoryVT().getRawBits()); |
0 |
893 |
ID.AddInteger(AT->getMemoryVT().getRawBits()); |
0 |
| 894 |
ID.AddInteger(AT->getRawSubclassData()); |
0 |
894 |
ID.AddInteger(AT->getRawSubclassData()); |
0 |
| 895 |
ID.AddInteger(AT->getPointerInfo().getAddrSpace()); |
0 |
895 |
ID.AddInteger(AT->getPointerInfo().getAddrSpace()); |
0 |
| 896 |
ID.AddInteger(AT->getMemOperand()->getFlags()); |
0 |
896 |
ID.AddInteger(AT->getMemOperand()->getFlags()); |
0 |
| 897 |
break; |
0 |
897 |
break; |
0 |
| 898 |
} |
--- |
898 |
} |
--- |
| 899 |
case ISD::VECTOR_SHUFFLE: { |
0 |
899 |
case ISD::VECTOR_SHUFFLE: { |
0 |
| 900 |
const ShuffleVectorSDNode *SVN = cast(N); |
0 |
900 |
const ShuffleVectorSDNode *SVN = cast(N); |
0 |
| 901 |
for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements(); |
0 |
901 |
for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements(); |
0 |
| 902 |
i != e; ++i) |
0 |
902 |
i != e; ++i) |
0 |
| 903 |
ID.AddInteger(SVN->getMaskElt(i)); |
0 |
903 |
ID.AddInteger(SVN->getMaskElt(i)); |
0 |
| 904 |
break; |
0 |
904 |
break; |
0 |
| 905 |
} |
--- |
905 |
} |
--- |
| 906 |
case ISD::TargetBlockAddress: |
0 |
906 |
case ISD::TargetBlockAddress: |
0 |
| 907 |
case ISD::BlockAddress: { |
--- |
907 |
case ISD::BlockAddress: { |
--- |
| 908 |
const BlockAddressSDNode *BA = cast(N); |
0 |
908 |
const BlockAddressSDNode *BA = cast(N); |
0 |
| 909 |
ID.AddPointer(BA->getBlockAddress()); |
0 |
909 |
ID.AddPointer(BA->getBlockAddress()); |
0 |
| 910 |
ID.AddInteger(BA->getOffset()); |
0 |
910 |
ID.AddInteger(BA->getOffset()); |
0 |
| 911 |
ID.AddInteger(BA->getTargetFlags()); |
0 |
911 |
ID.AddInteger(BA->getTargetFlags()); |
0 |
| 912 |
break; |
0 |
912 |
break; |
0 |
| 913 |
} |
--- |
913 |
} |
--- |
| 914 |
case ISD::AssertAlign: |
0 |
914 |
case ISD::AssertAlign: |
0 |
| 915 |
ID.AddInteger(cast(N)->getAlign().value()); |
0 |
915 |
ID.AddInteger(cast(N)->getAlign().value()); |
0 |
| 916 |
break; |
0 |
916 |
break; |
0 |
| 917 |
case ISD::PREFETCH: |
0 |
917 |
case ISD::PREFETCH: |
0 |
| 918 |
case ISD::INTRINSIC_VOID: |
--- |
918 |
case ISD::INTRINSIC_VOID: |
--- |
| 919 |
case ISD::INTRINSIC_W_CHAIN: |
--- |
919 |
case ISD::INTRINSIC_W_CHAIN: |
--- |
| 920 |
// Handled by MemIntrinsicSDNode check after the switch. |
--- |
920 |
// Handled by MemIntrinsicSDNode check after the switch. |
--- |
| 921 |
break; |
0 |
921 |
break; |
0 |
| 922 |
} // end switch (N->getOpcode()) |
--- |
922 |
} // end switch (N->getOpcode()) |
--- |
| 923 |
|
--- |
923 |
|
--- |
| 924 |
// MemIntrinsic nodes could also have subclass data, address spaces, and flags |
--- |
924 |
// MemIntrinsic nodes could also have subclass data, address spaces, and flags |
--- |
| 925 |
// to check. |
--- |
925 |
// to check. |
--- |
| 926 |
if (auto *MN = dyn_cast(N)) { |
37 |
926 |
if (auto *MN = dyn_cast(N)) { |
50 |
| 927 |
ID.AddInteger(MN->getRawSubclassData()); |
0 |
927 |
ID.AddInteger(MN->getRawSubclassData()); |
0 |
| 928 |
ID.AddInteger(MN->getPointerInfo().getAddrSpace()); |
0 |
928 |
ID.AddInteger(MN->getPointerInfo().getAddrSpace()); |
0 |
| 929 |
ID.AddInteger(MN->getMemOperand()->getFlags()); |
0 |
929 |
ID.AddInteger(MN->getMemOperand()->getFlags()); |
0 |
| 930 |
ID.AddInteger(MN->getMemoryVT().getRawBits()); |
0 |
930 |
ID.AddInteger(MN->getMemoryVT().getRawBits()); |
0 |
| 931 |
} |
--- |
931 |
} |
--- |
| 932 |
} |
37 |
932 |
} |
50 |
| 933 |
|
--- |
933 |
|
--- |
| 934 |
/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID |
--- |
934 |
/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID |
--- |
| 935 |
/// data. |
--- |
935 |
/// data. |
--- |
| 936 |
static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { |
36 |
936 |
static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { |
49 |
| 937 |
AddNodeIDOpcode(ID, N->getOpcode()); |
36 |
937 |
AddNodeIDOpcode(ID, N->getOpcode()); |
49 |
| 938 |
// Add the return value info. |
--- |
938 |
// Add the return value info. |
--- |
| 939 |
AddNodeIDValueTypes(ID, N->getVTList()); |
36 |
939 |
AddNodeIDValueTypes(ID, N->getVTList()); |
49 |
| 940 |
// Add the operand info. |
--- |
940 |
// Add the operand info. |
--- |
| 941 |
AddNodeIDOperands(ID, N->ops()); |
36 |
941 |
AddNodeIDOperands(ID, N->ops()); |
49 |
| 942 |
|
--- |
942 |
|
--- |
| 943 |
// Handle SDNode leafs with special info. |
--- |
943 |
// Handle SDNode leafs with special info. |
--- |
| 944 |
AddNodeIDCustom(ID, N); |
36 |
944 |
AddNodeIDCustom(ID, N); |
49 |
| 945 |
} |
36 |
945 |
} |
49 |
| 946 |
|
--- |
946 |
|
--- |
| 947 |
//===----------------------------------------------------------------------===// |
--- |
947 |
//===----------------------------------------------------------------------===// |
--- |
| 948 |
// SelectionDAG Class |
--- |
948 |
// SelectionDAG Class |
--- |
| 949 |
//===----------------------------------------------------------------------===// |
--- |
949 |
//===----------------------------------------------------------------------===// |
--- |
| 950 |
|
--- |
950 |
|
--- |
| 951 |
/// doNotCSE - Return true if CSE should not be performed for this node. |
--- |
951 |
/// doNotCSE - Return true if CSE should not be performed for this node. |
--- |
| 952 |
static bool doNotCSE(SDNode *N) { |
4 |
952 |
static bool doNotCSE(SDNode *N) { |
4 |
| 953 |
if (N->getValueType(0) == MVT::Glue) |
4 |
953 |
if (N->getValueType(0) == MVT::Glue) |
4 |
| 954 |
return true; // Never CSE anything that produces a flag. |
0 |
954 |
return true; // Never CSE anything that produces a flag. |
0 |
| 955 |
|
--- |
955 |
|
--- |
| 956 |
switch (N->getOpcode()) { |
4 |
956 |
switch (N->getOpcode()) { |
4 |
| 957 |
default: break; |
4 |
957 |
default: break; |
4 |
| 958 |
case ISD::HANDLENODE: |
0 |
958 |
case ISD::HANDLENODE: |
0 |
| 959 |
case ISD::EH_LABEL: |
--- |
959 |
case ISD::EH_LABEL: |
--- |
| 960 |
return true; // Never CSE these nodes. |
0 |
960 |
return true; // Never CSE these nodes. |
0 |
| 961 |
} |
--- |
961 |
} |
--- |
| 962 |
|
--- |
962 |
|
--- |
| 963 |
// Check that remaining values produced are not flags. |
--- |
963 |
// Check that remaining values produced are not flags. |
--- |
| 964 |
for (unsigned i = 1, e = N->getNumValues(); i != e; ++i) |
4 |
964 |
for (unsigned i = 1, e = N->getNumValues(); i != e; ++i) |
4 |
| 965 |
if (N->getValueType(i) == MVT::Glue) |
0 |
965 |
if (N->getValueType(i) == MVT::Glue) |
0 |
| 966 |
return true; // Never CSE anything that produces a flag. |
0 |
966 |
return true; // Never CSE anything that produces a flag. |
0 |
| 967 |
|
--- |
967 |
|
--- |
| 968 |
return false; |
4 |
968 |
return false; |
4 |
| 969 |
} |
--- |
969 |
} |
--- |
| 970 |
|
--- |
970 |
|
--- |
| 971 |
/// RemoveDeadNodes - This method deletes all unreachable nodes in the |
--- |
971 |
/// RemoveDeadNodes - This method deletes all unreachable nodes in the |
--- |
| 972 |
/// SelectionDAG. |
--- |
972 |
/// SelectionDAG. |
--- |
| 973 |
void SelectionDAG::RemoveDeadNodes() { |
17 |
973 |
void SelectionDAG::RemoveDeadNodes() { |
17 |
| 974 |
// Create a dummy node (which is not added to allnodes), that adds a reference |
--- |
974 |
// Create a dummy node (which is not added to allnodes), that adds a reference |
--- |
| 975 |
// to the root node, preventing it from being deleted. |
--- |
975 |
// to the root node, preventing it from being deleted. |
--- |
| 976 |
HandleSDNode Dummy(getRoot()); |
17 |
976 |
HandleSDNode Dummy(getRoot()); |
17 |
| 977 |
|
--- |
977 |
|
--- |
| 978 |
SmallVector DeadNodes; |
17 |
978 |
SmallVector DeadNodes; |
17 |
| 979 |
|
--- |
979 |
|
--- |
| 980 |
// Add all obviously-dead nodes to the DeadNodes worklist. |
--- |
980 |
// Add all obviously-dead nodes to the DeadNodes worklist. |
--- |
| 981 |
for (SDNode &Node : allnodes()) |
203 |
981 |
for (SDNode &Node : allnodes()) |
203 |
| 982 |
if (Node.use_empty()) |
186 |
982 |
if (Node.use_empty()) |
186 |
| 983 |
DeadNodes.push_back(&Node); |
1 |
983 |
DeadNodes.push_back(&Node); |
1 |
| 984 |
|
--- |
984 |
|
--- |
| 985 |
RemoveDeadNodes(DeadNodes); |
17 |
985 |
RemoveDeadNodes(DeadNodes); |
17 |
| 986 |
|
--- |
986 |
|
--- |
| 987 |
// If the root changed (e.g. it was a dead load, update the root). |
--- |
987 |
// If the root changed (e.g. it was a dead load, update the root). |
--- |
| 988 |
setRoot(Dummy.getValue()); |
17 |
988 |
setRoot(Dummy.getValue()); |
17 |
| 989 |
} |
17 |
989 |
} |
17 |
| 990 |
|
--- |
990 |
|
--- |
| 991 |
/// RemoveDeadNodes - This method deletes the unreachable nodes in the |
--- |
991 |
/// RemoveDeadNodes - This method deletes the unreachable nodes in the |
--- |
| 992 |
/// given list, and any nodes that become unreachable as a result. |
--- |
992 |
/// given list, and any nodes that become unreachable as a result. |
--- |
| 993 |
void SelectionDAG::RemoveDeadNodes(SmallVectorImpl &DeadNodes) { |
29 |
993 |
void SelectionDAG::RemoveDeadNodes(SmallVectorImpl &DeadNodes) { |
29 |
| 994 |
|
--- |
994 |
|
--- |
| 995 |
// Process the worklist, deleting the nodes and adding their uses to the |
--- |
995 |
// Process the worklist, deleting the nodes and adding their uses to the |
--- |
| 996 |
// worklist. |
--- |
996 |
// worklist. |
--- |
| 997 |
while (!DeadNodes.empty()) { |
44 |
997 |
while (!DeadNodes.empty()) { |
44 |
| 998 |
SDNode *N = DeadNodes.pop_back_val(); |
15 |
998 |
SDNode *N = DeadNodes.pop_back_val(); |
15 |
| 999 |
// Skip to next node if we've already managed to delete the node. This could |
--- |
999 |
// Skip to next node if we've already managed to delete the node. This could |
--- |
| 1000 |
// happen if replacing a node causes a node previously added to the node to |
--- |
1000 |
// happen if replacing a node causes a node previously added to the node to |
--- |
| 1001 |
// be deleted. |
--- |
1001 |
// be deleted. |
--- |
| 1002 |
if (N->getOpcode() == ISD::DELETED_NODE) |
15 |
1002 |
if (N->getOpcode() == ISD::DELETED_NODE) |
15 |
| 1003 |
continue; |
0 |
1003 |
continue; |
0 |
| 1004 |
|
--- |
1004 |
|
--- |
| 1005 |
for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) |
41 |
1005 |
for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) |
41 |
| 1006 |
DUL->NodeDeleted(N, nullptr); |
26 |
1006 |
DUL->NodeDeleted(N, nullptr); |
26 |
| 1007 |
|
--- |
1007 |
|
--- |
| 1008 |
// Take the node out of the appropriate CSE map. |
--- |
1008 |
// Take the node out of the appropriate CSE map. |
--- |
| 1009 |
RemoveNodeFromCSEMaps(N); |
15 |
1009 |
RemoveNodeFromCSEMaps(N); |
15 |
| 1010 |
|
--- |
1010 |
|
--- |
| 1011 |
// Next, brutally remove the operand list. This is safe to do, as there are |
--- |
1011 |
// Next, brutally remove the operand list. This is safe to do, as there are |
--- |
| 1012 |
// no cycles in the graph. |
--- |
1012 |
// no cycles in the graph. |
--- |
| 1013 |
for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { |
22 |
1013 |
for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { |
22 |
| 1014 |
SDUse &Use = *I++; |
7 |
1014 |
SDUse &Use = *I++; |
7 |
| 1015 |
SDNode *Operand = Use.getNode(); |
7 |
1015 |
SDNode *Operand = Use.getNode(); |
7 |
| 1016 |
Use.set(SDValue()); |
7 |
1016 |
Use.set(SDValue()); |
7 |
| 1017 |
|
--- |
1017 |
|
--- |
| 1018 |
// Now that we removed this operand, see if there are no uses of it left. |
--- |
1018 |
// Now that we removed this operand, see if there are no uses of it left. |
--- |
| 1019 |
if (Operand->use_empty()) |
7 |
1019 |
if (Operand->use_empty()) |
7 |
| 1020 |
DeadNodes.push_back(Operand); |
2 |
1020 |
DeadNodes.push_back(Operand); |
2 |
| 1021 |
} |
--- |
1021 |
} |
--- |
| 1022 |
|
--- |
1022 |
|
--- |
| 1023 |
DeallocateNode(N); |
15 |
1023 |
DeallocateNode(N); |
15 |
| 1024 |
} |
--- |
1024 |
} |
--- |
| 1025 |
} |
29 |
1025 |
} |
29 |
| 1026 |
|
--- |
1026 |
|
--- |
| 1027 |
void SelectionDAG::RemoveDeadNode(SDNode *N){ |
0 |
1027 |
void SelectionDAG::RemoveDeadNode(SDNode *N){ |
0 |
| 1028 |
SmallVector DeadNodes(1, N); |
0 |
1028 |
SmallVector DeadNodes(1, N); |
0 |
| 1029 |
|
--- |
1029 |
|
--- |
| 1030 |
// Create a dummy node that adds a reference to the root node, preventing |
--- |
1030 |
// Create a dummy node that adds a reference to the root node, preventing |
--- |
| 1031 |
// it from being deleted. (This matters if the root is an operand of the |
--- |
1031 |
// it from being deleted. (This matters if the root is an operand of the |
--- |
| 1032 |
// dead node.) |
--- |
1032 |
// dead node.) |
--- |
| 1033 |
HandleSDNode Dummy(getRoot()); |
0 |
1033 |
HandleSDNode Dummy(getRoot()); |
0 |
| 1034 |
|
--- |
1034 |
|
--- |
| 1035 |
RemoveDeadNodes(DeadNodes); |
0 |
1035 |
RemoveDeadNodes(DeadNodes); |
0 |
| 1036 |
} |
0 |
1036 |
} |
0 |
| 1037 |
|
--- |
1037 |
|
--- |
| 1038 |
void SelectionDAG::DeleteNode(SDNode *N) { |
11 |
1038 |
void SelectionDAG::DeleteNode(SDNode *N) { |
11 |
| 1039 |
// First take this out of the appropriate CSE map. |
--- |
1039 |
// First take this out of the appropriate CSE map. |
--- |
| 1040 |
RemoveNodeFromCSEMaps(N); |
11 |
1040 |
RemoveNodeFromCSEMaps(N); |
11 |
| 1041 |
|
--- |
1041 |
|
--- |
| 1042 |
// Finally, remove uses due to operands of this node, remove from the |
--- |
1042 |
// Finally, remove uses due to operands of this node, remove from the |
--- |
| 1043 |
// AllNodes list, and delete the node. |
--- |
1043 |
// AllNodes list, and delete the node. |
--- |
| 1044 |
DeleteNodeNotInCSEMaps(N); |
11 |
1044 |
DeleteNodeNotInCSEMaps(N); |
11 |
| 1045 |
} |
11 |
1045 |
} |
11 |
| 1046 |
|
--- |
1046 |
|
--- |
| 1047 |
void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { |
11 |
1047 |
void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { |
11 |
| 1048 |
assert(N->getIterator() != AllNodes.begin() && |
11 |
1048 |
assert(N->getIterator() != AllNodes.begin() && |
11 |
| 1049 |
"Cannot delete the entry node!"); |
--- |
1049 |
"Cannot delete the entry node!"); |
--- |
| 1050 |
assert(N->use_empty() && "Cannot delete a node that is not dead!"); |
11 |
1050 |
assert(N->use_empty() && "Cannot delete a node that is not dead!"); |
11 |
| 1051 |
|
--- |
1051 |
|
--- |
| 1052 |
// Drop all of the operands and decrement used node's use counts. |
--- |
1052 |
// Drop all of the operands and decrement used node's use counts. |
--- |
| 1053 |
N->DropOperands(); |
11 |
1053 |
N->DropOperands(); |
11 |
| 1054 |
|
--- |
1054 |
|
--- |
| 1055 |
DeallocateNode(N); |
11 |
1055 |
DeallocateNode(N); |
11 |
| 1056 |
} |
11 |
1056 |
} |
11 |
| 1057 |
|
--- |
1057 |
|
--- |
| 1058 |
void SDDbgInfo::add(SDDbgValue *V, bool isParameter) { |
0 |
1058 |
void SDDbgInfo::add(SDDbgValue *V, bool isParameter) { |
0 |
| 1059 |
assert(!(V->isVariadic() && isParameter)); |
0 |
1059 |
assert(!(V->isVariadic() && isParameter)); |
0 |
| 1060 |
if (isParameter) |
0 |
1060 |
if (isParameter) |
0 |
| 1061 |
ByvalParmDbgValues.push_back(V); |
0 |
1061 |
ByvalParmDbgValues.push_back(V); |
0 |
| 1062 |
else |
--- |
1062 |
else |
--- |
| 1063 |
DbgValues.push_back(V); |
0 |
1063 |
DbgValues.push_back(V); |
0 |
| 1064 |
for (const SDNode *Node : V->getSDNodes()) |
0 |
1064 |
for (const SDNode *Node : V->getSDNodes()) |
0 |
| 1065 |
if (Node) |
0 |
1065 |
if (Node) |
0 |
| 1066 |
DbgValMap[Node].push_back(V); |
0 |
1066 |
DbgValMap[Node].push_back(V); |
0 |
| 1067 |
} |
0 |
1067 |
} |
0 |
| 1068 |
|
--- |
1068 |
|
--- |
| 1069 |
void SDDbgInfo::erase(const SDNode *Node) { |
63 |
1069 |
void SDDbgInfo::erase(const SDNode *Node) { |
63 |
| 1070 |
DbgValMapType::iterator I = DbgValMap.find(Node); |
63 |
1070 |
DbgValMapType::iterator I = DbgValMap.find(Node); |
63 |
| 1071 |
if (I == DbgValMap.end()) |
63 |
1071 |
if (I == DbgValMap.end()) |
63 |
| 1072 |
return; |
63 |
1072 |
return; |
63 |
| 1073 |
for (auto &Val: I->second) |
0 |
1073 |
for (auto &Val: I->second) |
0 |
| 1074 |
Val->setIsInvalidated(); |
0 |
1074 |
Val->setIsInvalidated(); |
0 |
| 1075 |
DbgValMap.erase(I); |
0 |
1075 |
DbgValMap.erase(I); |
0 |
| 1076 |
} |
--- |
1076 |
} |
--- |
| 1077 |
|
--- |
1077 |
|
--- |
| 1078 |
void SelectionDAG::DeallocateNode(SDNode *N) { |
63 |
1078 |
void SelectionDAG::DeallocateNode(SDNode *N) { |
63 |
| 1079 |
// If we have operands, deallocate them. |
--- |
1079 |
// If we have operands, deallocate them. |
--- |
| 1080 |
removeOperands(N); |
63 |
1080 |
removeOperands(N); |
63 |
| 1081 |
|
--- |
1081 |
|
--- |
| 1082 |
NodeAllocator.Deallocate(AllNodes.remove(N)); |
63 |
1082 |
NodeAllocator.Deallocate(AllNodes.remove(N)); |
63 |
| 1083 |
|
--- |
1083 |
|
--- |
| 1084 |
// Set the opcode to DELETED_NODE to help catch bugs when node |
--- |
1084 |
// Set the opcode to DELETED_NODE to help catch bugs when node |
--- |
| 1085 |
// memory is reallocated. |
--- |
1085 |
// memory is reallocated. |
--- |
| 1086 |
// FIXME: There are places in SDag that have grown a dependency on the opcode |
--- |
1086 |
// FIXME: There are places in SDag that have grown a dependency on the opcode |
--- |
| 1087 |
// value in the released node. |
--- |
1087 |
// value in the released node. |
--- |
| 1088 |
__asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType)); |
--- |
1088 |
__asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType)); |
--- |
| 1089 |
N->NodeType = ISD::DELETED_NODE; |
63 |
1089 |
N->NodeType = ISD::DELETED_NODE; |
63 |
| 1090 |
|
--- |
1090 |
|
--- |
| 1091 |
// If any of the SDDbgValue nodes refer to this SDNode, invalidate |
--- |
1091 |
// If any of the SDDbgValue nodes refer to this SDNode, invalidate |
--- |
| 1092 |
// them and forget about that node. |
--- |
1092 |
// them and forget about that node. |
--- |
| 1093 |
DbgInfo->erase(N); |
63 |
1093 |
DbgInfo->erase(N); |
63 |
| 1094 |
|
--- |
1094 |
|
--- |
| 1095 |
// Invalidate extra info. |
--- |
1095 |
// Invalidate extra info. |
--- |
| 1096 |
SDEI.erase(N); |
63 |
1096 |
SDEI.erase(N); |
63 |
| 1097 |
} |
63 |
1097 |
} |
63 |
| 1098 |
|
--- |
1098 |
|
--- |
| 1099 |
#ifndef NDEBUG |
--- |
1099 |
#ifndef NDEBUG |
--- |
| 1100 |
/// VerifySDNode - Check the given SDNode. Aborts if it is invalid. |
--- |
1100 |
/// VerifySDNode - Check the given SDNode. Aborts if it is invalid. |
--- |
| 1101 |
static void VerifySDNode(SDNode *N) { |
69 |
1101 |
static void VerifySDNode(SDNode *N) { |
69 |
| 1102 |
switch (N->getOpcode()) { |
69 |
1102 |
switch (N->getOpcode()) { |
69 |
| 1103 |
default: |
69 |
1103 |
default: |
69 |
| 1104 |
break; |
69 |
1104 |
break; |
69 |
| 1105 |
case ISD::BUILD_PAIR: { |
0 |
1105 |
case ISD::BUILD_PAIR: { |
0 |
| 1106 |
EVT VT = N->getValueType(0); |
0 |
1106 |
EVT VT = N->getValueType(0); |
0 |
| 1107 |
assert(N->getNumValues() == 1 && "Too many results!"); |
0 |
1107 |
assert(N->getNumValues() == 1 && "Too many results!"); |
0 |
| 1108 |
assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && |
0 |
1108 |
assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && |
0 |
| 1109 |
"Wrong return type!"); |
--- |
1109 |
"Wrong return type!"); |
--- |
| 1110 |
assert(N->getNumOperands() == 2 && "Wrong number of operands!"); |
0 |
1110 |
assert(N->getNumOperands() == 2 && "Wrong number of operands!"); |
0 |
| 1111 |
assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && |
0 |
1111 |
assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && |
0 |
| 1112 |
"Mismatched operand types!"); |
--- |
1112 |
"Mismatched operand types!"); |
--- |
| 1113 |
assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() && |
0 |
1113 |
assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() && |
0 |
| 1114 |
"Wrong operand type!"); |
--- |
1114 |
"Wrong operand type!"); |
--- |
| 1115 |
assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && |
0 |
1115 |
assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && |
0 |
| 1116 |
"Wrong return type size"); |
--- |
1116 |
"Wrong return type size"); |
--- |
| 1117 |
break; |
0 |
1117 |
break; |
0 |
| 1118 |
} |
--- |
1118 |
} |
--- |
| 1119 |
case ISD::BUILD_VECTOR: { |
0 |
1119 |
case ISD::BUILD_VECTOR: { |
0 |
| 1120 |
assert(N->getNumValues() == 1 && "Too many results!"); |
0 |
1120 |
assert(N->getNumValues() == 1 && "Too many results!"); |
0 |
| 1121 |
assert(N->getValueType(0).isVector() && "Wrong return type!"); |
0 |
1121 |
assert(N->getValueType(0).isVector() && "Wrong return type!"); |
0 |
| 1122 |
assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && |
0 |
1122 |
assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && |
0 |
| 1123 |
"Wrong number of operands!"); |
--- |
1123 |
"Wrong number of operands!"); |
--- |
| 1124 |
EVT EltVT = N->getValueType(0).getVectorElementType(); |
0 |
1124 |
EVT EltVT = N->getValueType(0).getVectorElementType(); |
0 |
| 1125 |
for (const SDUse &Op : N->ops()) { |
0 |
1125 |
for (const SDUse &Op : N->ops()) { |
0 |
| 1126 |
assert((Op.getValueType() == EltVT || |
0 |
1126 |
assert((Op.getValueType() == EltVT || |
0 |
| 1127 |
(EltVT.isInteger() && Op.getValueType().isInteger() && |
--- |
1127 |
(EltVT.isInteger() && Op.getValueType().isInteger() && |
--- |
| 1128 |
EltVT.bitsLE(Op.getValueType()))) && |
--- |
1128 |
EltVT.bitsLE(Op.getValueType()))) && |
--- |
| 1129 |
"Wrong operand type!"); |
--- |
1129 |
"Wrong operand type!"); |
--- |
| 1130 |
assert(Op.getValueType() == N->getOperand(0).getValueType() && |
0 |
1130 |
assert(Op.getValueType() == N->getOperand(0).getValueType() && |
0 |
| 1131 |
"Operands must all have the same type"); |
--- |
1131 |
"Operands must all have the same type"); |
--- |
| 1132 |
} |
--- |
1132 |
} |
--- |
| 1133 |
break; |
0 |
1133 |
break; |
0 |
| 1134 |
} |
--- |
1134 |
} |
--- |
| 1135 |
} |
--- |
1135 |
} |
--- |
| 1136 |
} |
69 |
1136 |
} |
69 |
| 1137 |
#endif // NDEBUG |
--- |
1137 |
#endif // NDEBUG |
--- |
| 1138 |
|
--- |
1138 |
|
--- |
| 1139 |
/// Insert a newly allocated node into the DAG. |
--- |
1139 |
/// Insert a newly allocated node into the DAG. |
--- |
| 1140 |
/// |
--- |
1140 |
/// |
--- |
| 1141 |
/// Handles insertion into the all nodes list and CSE map, as well as |
--- |
1141 |
/// Handles insertion into the all nodes list and CSE map, as well as |
--- |
| 1142 |
/// verification and other common operations when a new node is allocated. |
--- |
1142 |
/// verification and other common operations when a new node is allocated. |
--- |
| 1143 |
void SelectionDAG::InsertNode(SDNode *N) { |
69 |
1143 |
void SelectionDAG::InsertNode(SDNode *N) { |
69 |
| 1144 |
AllNodes.push_back(N); |
69 |
1144 |
AllNodes.push_back(N); |
69 |
| 1145 |
#ifndef NDEBUG |
--- |
1145 |
#ifndef NDEBUG |
--- |
| 1146 |
N->PersistentId = NextPersistentId++; |
69 |
1146 |
N->PersistentId = NextPersistentId++; |
69 |
| 1147 |
VerifySDNode(N); |
69 |
1147 |
VerifySDNode(N); |
69 |
| 1148 |
#endif |
--- |
1148 |
#endif |
--- |
| 1149 |
for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) |
88 |
1149 |
for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) |
88 |
| 1150 |
DUL->NodeInserted(N); |
19 |
1150 |
DUL->NodeInserted(N); |
19 |
| 1151 |
} |
69 |
1151 |
} |
69 |
| 1152 |
|
--- |
1152 |
|
--- |
| 1153 |
/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that |
--- |
1153 |
/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that |
--- |
| 1154 |
/// correspond to it. This is useful when we're about to delete or repurpose |
--- |
1154 |
/// correspond to it. This is useful when we're about to delete or repurpose |
--- |
| 1155 |
/// the node. We don't want future request for structurally identical nodes |
--- |
1155 |
/// the node. We don't want future request for structurally identical nodes |
--- |
| 1156 |
/// to return N anymore. |
--- |
1156 |
/// to return N anymore. |
--- |
| 1157 |
bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { |
44 |
1157 |
bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { |
44 |
| 1158 |
bool Erased = false; |
44 |
1158 |
bool Erased = false; |
44 |
| 1159 |
switch (N->getOpcode()) { |
44 |
1159 |
switch (N->getOpcode()) { |
44 |
| 1160 |
case ISD::HANDLENODE: return false; // noop. |
0 |
1160 |
case ISD::HANDLENODE: return false; // noop. |
0 |
| 1161 |
case ISD::CONDCODE: |
2 |
1161 |
case ISD::CONDCODE: |
2 |
| 1162 |
assert(CondCodeNodes[cast(N)->get()] && |
2 |
1162 |
assert(CondCodeNodes[cast(N)->get()] && |
2 |
| 1163 |
"Cond code doesn't exist!"); |
--- |
1163 |
"Cond code doesn't exist!"); |
--- |
| 1164 |
Erased = CondCodeNodes[cast(N)->get()] != nullptr; |
2 |
1164 |
Erased = CondCodeNodes[cast(N)->get()] != nullptr; |
2 |
| 1165 |
CondCodeNodes[cast(N)->get()] = nullptr; |
2 |
1165 |
CondCodeNodes[cast(N)->get()] = nullptr; |
2 |
| 1166 |
break; |
2 |
1166 |
break; |
2 |
| 1167 |
case ISD::ExternalSymbol: |
0 |
1167 |
case ISD::ExternalSymbol: |
0 |
| 1168 |
Erased = ExternalSymbols.erase(cast(N)->getSymbol()); |
0 |
1168 |
Erased = ExternalSymbols.erase(cast(N)->getSymbol()); |
0 |
| 1169 |
break; |
0 |
1169 |
break; |
0 |
| 1170 |
case ISD::TargetExternalSymbol: { |
0 |
1170 |
case ISD::TargetExternalSymbol: { |
0 |
| 1171 |
ExternalSymbolSDNode *ESN = cast(N); |
0 |
1171 |
ExternalSymbolSDNode *ESN = cast(N); |
0 |
| 1172 |
Erased = TargetExternalSymbols.erase(std::pair( |
0 |
1172 |
Erased = TargetExternalSymbols.erase(std::pair( |
0 |
| 1173 |
ESN->getSymbol(), ESN->getTargetFlags())); |
0 |
1173 |
ESN->getSymbol(), ESN->getTargetFlags())); |
0 |
| 1174 |
break; |
0 |
1174 |
break; |
0 |
| 1175 |
} |
--- |
1175 |
} |
--- |
| 1176 |
case ISD::MCSymbol: { |
0 |
1176 |
case ISD::MCSymbol: { |
0 |
| 1177 |
auto *MCSN = cast(N); |
0 |
1177 |
auto *MCSN = cast(N); |
0 |
| 1178 |
Erased = MCSymbols.erase(MCSN->getMCSymbol()); |
0 |
1178 |
Erased = MCSymbols.erase(MCSN->getMCSymbol()); |
0 |
| 1179 |
break; |
0 |
1179 |
break; |
0 |
| 1180 |
} |
--- |
1180 |
} |
--- |
| 1181 |
case ISD::VALUETYPE: { |
0 |
1181 |
case ISD::VALUETYPE: { |
0 |
| 1182 |
EVT VT = cast(N)->getVT(); |
0 |
1182 |
EVT VT = cast(N)->getVT(); |
0 |
| 1183 |
if (VT.isExtended()) { |
0 |
1183 |
if (VT.isExtended()) { |
0 |
| 1184 |
Erased = ExtendedValueTypeNodes.erase(VT); |
0 |
1184 |
Erased = ExtendedValueTypeNodes.erase(VT); |
0 |
| 1185 |
} else { |
--- |
1185 |
} else { |
--- |
| 1186 |
Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr; |
0 |
1186 |
Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr; |
0 |
| 1187 |
ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr; |
0 |
1187 |
ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr; |
0 |
| 1188 |
} |
--- |
1188 |
} |
--- |
| 1189 |
break; |
0 |
1189 |
break; |
0 |
| 1190 |
} |
--- |
1190 |
} |
--- |
| 1191 |
default: |
42 |
1191 |
default: |
42 |
| 1192 |
// Remove it from the CSE Map. |
--- |
1192 |
// Remove it from the CSE Map. |
--- |
| 1193 |
assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); |
42 |
1193 |
assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); |
42 |
| 1194 |
assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); |
42 |
1194 |
assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); |
42 |
| 1195 |
Erased = CSEMap.RemoveNode(N); |
42 |
1195 |
Erased = CSEMap.RemoveNode(N); |
42 |
| 1196 |
break; |
42 |
1196 |
break; |
42 |
| 1197 |
} |
--- |
1197 |
} |
--- |
| 1198 |
#ifndef NDEBUG |
--- |
1198 |
#ifndef NDEBUG |
--- |
| 1199 |
// Verify that the node was actually in one of the CSE maps, unless it has a |
--- |
1199 |
// Verify that the node was actually in one of the CSE maps, unless it has a |
--- |
| 1200 |
// flag result (which cannot be CSE'd) or is one of the special cases that are |
--- |
1200 |
// flag result (which cannot be CSE'd) or is one of the special cases that are |
--- |
| 1201 |
// not subject to CSE. |
--- |
1201 |
// not subject to CSE. |
--- |
| 1202 |
if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue && |
0 |
1202 |
if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue && |
0 |
| 1203 |
!N->isMachineOpcode() && !doNotCSE(N)) { |
44 |
1203 |
!N->isMachineOpcode() && !doNotCSE(N)) { |
44 |
| 1204 |
N->dump(this); |
0 |
1204 |
N->dump(this); |
0 |
| 1205 |
dbgs() << "\n"; |
0 |
1205 |
dbgs() << "\n"; |
0 |
| 1206 |
llvm_unreachable("Node is not in map!"); |
0 |
1206 |
llvm_unreachable("Node is not in map!"); |
0 |
| 1207 |
} |
--- |
1207 |
} |
--- |
| 1208 |
#endif |
--- |
1208 |
#endif |
--- |
| 1209 |
return Erased; |
44 |
1209 |
return Erased; |
44 |
| 1210 |
} |
--- |
1210 |
} |
--- |
| 1211 |
|
--- |
1211 |
|
--- |
| 1212 |
/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE |
--- |
1212 |
/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE |
--- |
| 1213 |
/// maps and modified in place. Add it back to the CSE maps, unless an identical |
--- |
1213 |
/// maps and modified in place. Add it back to the CSE maps, unless an identical |
--- |
| 1214 |
/// node already exists, in which case transfer all its users to the existing |
--- |
1214 |
/// node already exists, in which case transfer all its users to the existing |
--- |
| 1215 |
/// node. This transfer can potentially trigger recursive merging. |
--- |
1215 |
/// node. This transfer can potentially trigger recursive merging. |
--- |
| 1216 |
void |
--- |
1216 |
void |
--- |
| 1217 |
SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) { |
3 |
1217 |
SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) { |
3 |
| 1218 |
// For node types that aren't CSE'd, just act as if no identical node |
--- |
1218 |
// For node types that aren't CSE'd, just act as if no identical node |
--- |
| 1219 |
// already exists. |
--- |
1219 |
// already exists. |
--- |
| 1220 |
if (!doNotCSE(N)) { |
3 |
1220 |
if (!doNotCSE(N)) { |
3 |
| 1221 |
SDNode *Existing = CSEMap.GetOrInsertNode(N); |
3 |
1221 |
SDNode *Existing = CSEMap.GetOrInsertNode(N); |
3 |
| 1222 |
if (Existing != N) { |
3 |
1222 |
if (Existing != N) { |
3 |
| 1223 |
// If there was already an existing matching node, use ReplaceAllUsesWith |
--- |
1223 |
// If there was already an existing matching node, use ReplaceAllUsesWith |
--- |
| 1224 |
// to replace the dead one with the existing one. This can cause |
--- |
1224 |
// to replace the dead one with the existing one. This can cause |
--- |
| 1225 |
// recursive merging of other unrelated nodes down the line. |
--- |
1225 |
// recursive merging of other unrelated nodes down the line. |
--- |
| 1226 |
ReplaceAllUsesWith(N, Existing); |
0 |
1226 |
ReplaceAllUsesWith(N, Existing); |
0 |
| 1227 |
|
--- |
1227 |
|
--- |
| 1228 |
// N is now dead. Inform the listeners and delete it. |
--- |
1228 |
// N is now dead. Inform the listeners and delete it. |
--- |
| 1229 |
for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) |
0 |
1229 |
for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) |
0 |
| 1230 |
DUL->NodeDeleted(N, Existing); |
0 |
1230 |
DUL->NodeDeleted(N, Existing); |
0 |
| 1231 |
DeleteNodeNotInCSEMaps(N); |
0 |
1231 |
DeleteNodeNotInCSEMaps(N); |
0 |
| 1232 |
return; |
0 |
1232 |
return; |
0 |
| 1233 |
} |
--- |
1233 |
} |
--- |
| 1234 |
} |
--- |
1234 |
} |
--- |
| 1235 |
|
--- |
1235 |
|
--- |
| 1236 |
// If the node doesn't already exist, we updated it. Inform listeners. |
--- |
1236 |
// If the node doesn't already exist, we updated it. Inform listeners. |
--- |
| 1237 |
for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) |
11 |
1237 |
for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) |
11 |
| 1238 |
DUL->NodeUpdated(N); |
8 |
1238 |
DUL->NodeUpdated(N); |
8 |
| 1239 |
} |
--- |
1239 |
} |
--- |
| 1240 |
|
--- |
1240 |
|
--- |
| 1241 |
/// FindModifiedNodeSlot - Find a slot for the specified node if its operands |
--- |
1241 |
/// FindModifiedNodeSlot - Find a slot for the specified node if its operands |
--- |
| 1242 |
/// were replaced with those specified. If this node is never memoized, |
--- |
1242 |
/// were replaced with those specified. If this node is never memoized, |
--- |
| 1243 |
/// return null, otherwise return a pointer to the slot it would take. If a |
--- |
1243 |
/// return null, otherwise return a pointer to the slot it would take. If a |
--- |
| 1244 |
/// node already exists with these operands, the slot will be non-null. |
--- |
1244 |
/// node already exists with these operands, the slot will be non-null. |
--- |
| 1245 |
SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, |
0 |
1245 |
SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, |
0 |
| 1246 |
void *&InsertPos) { |
--- |
1246 |
void *&InsertPos) { |
--- |
| 1247 |
if (doNotCSE(N)) |
0 |
1247 |
if (doNotCSE(N)) |
0 |
| 1248 |
return nullptr; |
0 |
1248 |
return nullptr; |
0 |
| 1249 |
|
--- |
1249 |
|
--- |
| 1250 |
SDValue Ops[] = { Op }; |
0 |
1250 |
SDValue Ops[] = { Op }; |
0 |
| 1251 |
FoldingSetNodeID ID; |
0 |
1251 |
FoldingSetNodeID ID; |
0 |
| 1252 |
AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); |
0 |
1252 |
AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); |
0 |
| 1253 |
AddNodeIDCustom(ID, N); |
0 |
1253 |
AddNodeIDCustom(ID, N); |
0 |
| 1254 |
SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); |
0 |
1254 |
SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); |
0 |
| 1255 |
if (Node) |
0 |
1255 |
if (Node) |
0 |
| 1256 |
Node->intersectFlagsWith(N->getFlags()); |
0 |
1256 |
Node->intersectFlagsWith(N->getFlags()); |
0 |
| 1257 |
return Node; |
0 |
1257 |
return Node; |
0 |
| 1258 |
} |
0 |
1258 |
} |
0 |
| 1259 |
|
--- |
1259 |
|
--- |
| 1260 |
/// FindModifiedNodeSlot - Find a slot for the specified node if its operands |
--- |
1260 |
/// FindModifiedNodeSlot - Find a slot for the specified node if its operands |
--- |
| 1261 |
/// were replaced with those specified. If this node is never memoized, |
--- |
1261 |
/// were replaced with those specified. If this node is never memoized, |
--- |
| 1262 |
/// return null, otherwise return a pointer to the slot it would take. If a |
--- |
1262 |
/// return null, otherwise return a pointer to the slot it would take. If a |
--- |
| 1263 |
/// node already exists with these operands, the slot will be non-null. |
--- |
1263 |
/// node already exists with these operands, the slot will be non-null. |
--- |
| 1264 |
SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, |
0 |
1264 |
SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, |
0 |
| 1265 |
SDValue Op1, SDValue Op2, |
--- |
1265 |
SDValue Op1, SDValue Op2, |
--- |
| 1266 |
void *&InsertPos) { |
--- |
1266 |
void *&InsertPos) { |
--- |
| 1267 |
if (doNotCSE(N)) |
0 |
1267 |
if (doNotCSE(N)) |
0 |
| 1268 |
return nullptr; |
0 |
1268 |
return nullptr; |
0 |
| 1269 |
|
--- |
1269 |
|
--- |
| 1270 |
SDValue Ops[] = { Op1, Op2 }; |
0 |
1270 |
SDValue Ops[] = { Op1, Op2 }; |
0 |
| 1271 |
FoldingSetNodeID ID; |
0 |
1271 |
FoldingSetNodeID ID; |
0 |
| 1272 |
AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); |
0 |
1272 |
AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); |
0 |
| 1273 |
AddNodeIDCustom(ID, N); |
0 |
1273 |
AddNodeIDCustom(ID, N); |
0 |
| 1274 |
SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); |
0 |
1274 |
SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); |
0 |
| 1275 |
if (Node) |
0 |
1275 |
if (Node) |
0 |
| 1276 |
Node->intersectFlagsWith(N->getFlags()); |
0 |
1276 |
Node->intersectFlagsWith(N->getFlags()); |
0 |
| 1277 |
return Node; |
0 |
1277 |
return Node; |
0 |
| 1278 |
} |
0 |
1278 |
} |
0 |
| 1279 |
|
--- |
1279 |
|
--- |
| 1280 |
/// FindModifiedNodeSlot - Find a slot for the specified node if its operands |
--- |
1280 |
/// FindModifiedNodeSlot - Find a slot for the specified node if its operands |
--- |
| 1281 |
/// were replaced with those specified. If this node is never memoized, |
--- |
1281 |
/// were replaced with those specified. If this node is never memoized, |
--- |
| 1282 |
/// return null, otherwise return a pointer to the slot it would take. If a |
--- |
1282 |
/// return null, otherwise return a pointer to the slot it would take. If a |
--- |
| 1283 |
/// node already exists with these operands, the slot will be non-null. |
--- |
1283 |
/// node already exists with these operands, the slot will be non-null. |
--- |
| 1284 |
SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef Ops, |
1 |
1284 |
SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef Ops, |
1 |
| 1285 |
void *&InsertPos) { |
--- |
1285 |
void *&InsertPos) { |
--- |
| 1286 |
if (doNotCSE(N)) |
1 |
1286 |
if (doNotCSE(N)) |
1 |
| 1287 |
return nullptr; |
0 |
1287 |
return nullptr; |
0 |
| 1288 |
|
--- |
1288 |
|
--- |
| 1289 |
FoldingSetNodeID ID; |
1 |
1289 |
FoldingSetNodeID ID; |
1 |
| 1290 |
AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); |
1 |
1290 |
AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); |
1 |
| 1291 |
AddNodeIDCustom(ID, N); |
1 |
1291 |
AddNodeIDCustom(ID, N); |
1 |
| 1292 |
SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); |
1 |
1292 |
SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); |
1 |
| 1293 |
if (Node) |
1 |
1293 |
if (Node) |
1 |
| 1294 |
Node->intersectFlagsWith(N->getFlags()); |
0 |
1294 |
Node->intersectFlagsWith(N->getFlags()); |
0 |
| 1295 |
return Node; |
1 |
1295 |
return Node; |
1 |
| 1296 |
} |
1 |
1296 |
} |
1 |
| 1297 |
|
--- |
1297 |
|
--- |
| 1298 |
Align SelectionDAG::getEVTAlign(EVT VT) const { |
5 |
1298 |
Align SelectionDAG::getEVTAlign(EVT VT) const { |
5 |
| 1299 |
Type *Ty = VT == MVT::iPTR ? |
5 |
1299 |
Type *Ty = VT == MVT::iPTR ? |
5 |
| 1300 |
PointerType::get(Type::getInt8Ty(*getContext()), 0) : |
0 |
1300 |
PointerType::get(Type::getInt8Ty(*getContext()), 0) : |
0 |
| 1301 |
VT.getTypeForEVT(*getContext()); |
5 |
1301 |
VT.getTypeForEVT(*getContext()); |
5 |
| 1302 |
|
--- |
1302 |
|
--- |
| 1303 |
return getDataLayout().getABITypeAlign(Ty); |
5 |
1303 |
return getDataLayout().getABITypeAlign(Ty); |
5 |
| 1304 |
} |
--- |
1304 |
} |
--- |
| 1305 |
|
--- |
1305 |
|
--- |
| 1306 |
// EntryNode could meaningfully have debug info if we can find it... |
--- |
1306 |
// EntryNode could meaningfully have debug info if we can find it... |
--- |
| 1307 |
SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) |
2 |
1307 |
SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) |
2 |
| 1308 |
: TM(tm), OptLevel(OL), |
2 |
1308 |
: TM(tm), OptLevel(OL), |
2 |
| 1309 |
EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other, MVT::Glue)), |
2 |
1309 |
EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other, MVT::Glue)), |
2 |
| 1310 |
Root(getEntryNode()) { |
4 |
1310 |
Root(getEntryNode()) { |
4 |
| 1311 |
InsertNode(&EntryNode); |
2 |
1311 |
InsertNode(&EntryNode); |
2 |
| 1312 |
DbgInfo = new SDDbgInfo(); |
2 |
1312 |
DbgInfo = new SDDbgInfo(); |
2 |
| 1313 |
} |
2 |
1313 |
} |
2 |
| 1314 |
|
--- |
1314 |
|
--- |
| 1315 |
void SelectionDAG::init(MachineFunction &NewMF, |
1 |
1315 |
void SelectionDAG::init(MachineFunction &NewMF, |
1 |
| 1316 |
OptimizationRemarkEmitter &NewORE, Pass *PassPtr, |
--- |
1316 |
OptimizationRemarkEmitter &NewORE, Pass *PassPtr, |
--- |
| 1317 |
const TargetLibraryInfo *LibraryInfo, |
--- |
1317 |
const TargetLibraryInfo *LibraryInfo, |
--- |
| 1318 |
UniformityInfo *NewUA, ProfileSummaryInfo *PSIin, |
--- |
1318 |
UniformityInfo *NewUA, ProfileSummaryInfo *PSIin, |
--- |
| 1319 |
BlockFrequencyInfo *BFIin, |
--- |
1319 |
BlockFrequencyInfo *BFIin, |
--- |
| 1320 |
FunctionVarLocs const *VarLocs) { |
--- |
1320 |
FunctionVarLocs const *VarLocs) { |
--- |
| 1321 |
MF = &NewMF; |
1 |
1321 |
MF = &NewMF; |
1 |
| 1322 |
SDAGISelPass = PassPtr; |
1 |
1322 |
SDAGISelPass = PassPtr; |
1 |
| 1323 |
ORE = &NewORE; |
1 |
1323 |
ORE = &NewORE; |
1 |
| 1324 |
TLI = getSubtarget().getTargetLowering(); |
1 |
1324 |
TLI = getSubtarget().getTargetLowering(); |
1 |
| 1325 |
TSI = getSubtarget().getSelectionDAGInfo(); |
1 |
1325 |
TSI = getSubtarget().getSelectionDAGInfo(); |
1 |
| 1326 |
LibInfo = LibraryInfo; |
1 |
1326 |
LibInfo = LibraryInfo; |
1 |
| 1327 |
Context = &MF->getFunction().getContext(); |
1 |
1327 |
Context = &MF->getFunction().getContext(); |
1 |
| 1328 |
UA = NewUA; |
1 |
1328 |
UA = NewUA; |
1 |
| 1329 |
PSI = PSIin; |
1 |
1329 |
PSI = PSIin; |
1 |
| 1330 |
BFI = BFIin; |
1 |
1330 |
BFI = BFIin; |
1 |
| 1331 |
FnVarLocs = VarLocs; |
1 |
1331 |
FnVarLocs = VarLocs; |
1 |
| 1332 |
} |
1 |
1332 |
} |
1 |
| 1333 |
|
--- |
1333 |
|
--- |
| 1334 |
SelectionDAG::~SelectionDAG() { |
2 |
1334 |
SelectionDAG::~SelectionDAG() { |
2 |
| 1335 |
assert(!UpdateListeners && "Dangling registered DAGUpdateListeners"); |
2 |
1335 |
assert(!UpdateListeners && "Dangling registered DAGUpdateListeners"); |
2 |
| 1336 |
allnodes_clear(); |
2 |
1336 |
allnodes_clear(); |
2 |
| 1337 |
OperandRecycler.clear(OperandAllocator); |
2 |
1337 |
OperandRecycler.clear(OperandAllocator); |
2 |
| 1338 |
delete DbgInfo; |
2 |
1338 |
delete DbgInfo; |
2 |
| 1339 |
} |
2 |
1339 |
} |
2 |
| 1340 |
|
--- |
1340 |
|
--- |
| 1341 |
bool SelectionDAG::shouldOptForSize() const { |
9 |
1341 |
bool SelectionDAG::shouldOptForSize() const { |
9 |
| 1342 |
return MF->getFunction().hasOptSize() || |
18 |
1342 |
return MF->getFunction().hasOptSize() || |
18 |
| 1343 |
llvm::shouldOptimizeForSize(FLI->MBB->getBasicBlock(), PSI, BFI); |
18 |
1343 |
llvm::shouldOptimizeForSize(FLI->MBB->getBasicBlock(), PSI, BFI); |
18 |
| 1344 |
} |
--- |
1344 |
} |
--- |
| 1345 |
|
--- |
1345 |
|
--- |
| 1346 |
void SelectionDAG::allnodes_clear() { |
6 |
1346 |
void SelectionDAG::allnodes_clear() { |
6 |
| 1347 |
assert(&*AllNodes.begin() == &EntryNode); |
6 |
1347 |
assert(&*AllNodes.begin() == &EntryNode); |
6 |
| 1348 |
AllNodes.remove(AllNodes.begin()); |
6 |
1348 |
AllNodes.remove(AllNodes.begin()); |
6 |
| 1349 |
while (!AllNodes.empty()) |
43 |
1349 |
while (!AllNodes.empty()) |
43 |
| 1350 |
DeallocateNode(&AllNodes.front()); |
37 |
1350 |
DeallocateNode(&AllNodes.front()); |
37 |
| 1351 |
#ifndef NDEBUG |
--- |
1351 |
#ifndef NDEBUG |
--- |
| 1352 |
NextPersistentId = 0; |
6 |
1352 |
NextPersistentId = 0; |
6 |
| 1353 |
#endif |
--- |
1353 |
#endif |
--- |
| 1354 |
} |
6 |
1354 |
} |
6 |
| 1355 |
|
--- |
1355 |
|
--- |
| 1356 |
SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID, |
25 |
1356 |
SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID, |
25 |
| 1357 |
void *&InsertPos) { |
--- |
1357 |
void *&InsertPos) { |
--- |
| 1358 |
SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos); |
25 |
1358 |
SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos); |
25 |
| 1359 |
if (N) { |
25 |
1359 |
if (N) { |
25 |
| 1360 |
switch (N->getOpcode()) { |
3 |
1360 |
switch (N->getOpcode()) { |
3 |
| 1361 |
default: break; |
3 |
1361 |
default: break; |
3 |
| 1362 |
case ISD::Constant: |
0 |
1362 |
case ISD::Constant: |
0 |
| 1363 |
case ISD::ConstantFP: |
--- |
1363 |
case ISD::ConstantFP: |
--- |
| 1364 |
llvm_unreachable("Querying for Constant and ConstantFP nodes requires " |
0 |
1364 |
llvm_unreachable("Querying for Constant and ConstantFP nodes requires " |
0 |
| 1365 |
"debug location. Use another overload."); |
--- |
1365 |
"debug location. Use another overload."); |
--- |
| 1366 |
} |
--- |
1366 |
} |
--- |
| 1367 |
} |
--- |
1367 |
} |
--- |
| 1368 |
return N; |
25 |
1368 |
return N; |
25 |
| 1369 |
} |
--- |
1369 |
} |
--- |
| 1370 |
|
--- |
1370 |
|
--- |
| 1371 |
SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID, |
71 |
1371 |
SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID, |
71 |
| 1372 |
const SDLoc &DL, void *&InsertPos) { |
--- |
1372 |
const SDLoc &DL, void *&InsertPos) { |
--- |
| 1373 |
SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos); |
71 |
1373 |
SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos); |
71 |
| 1374 |
if (N) { |
71 |
1374 |
if (N) { |
71 |
| 1375 |
switch (N->getOpcode()) { |
15 |
1375 |
switch (N->getOpcode()) { |
15 |
| 1376 |
case ISD::Constant: |
5 |
1376 |
case ISD::Constant: |
5 |
| 1377 |
case ISD::ConstantFP: |
--- |
1377 |
case ISD::ConstantFP: |
--- |
| 1378 |
// Erase debug location from the node if the node is used at several |
--- |
1378 |
// Erase debug location from the node if the node is used at several |
--- |
| 1379 |
// different places. Do not propagate one location to all uses as it |
--- |
1379 |
// different places. Do not propagate one location to all uses as it |
--- |
| 1380 |
// will cause a worse single stepping debugging experience. |
--- |
1380 |
// will cause a worse single stepping debugging experience. |
--- |
| 1381 |
if (N->getDebugLoc() != DL.getDebugLoc()) |
5 |
1381 |
if (N->getDebugLoc() != DL.getDebugLoc()) |
5 |
| 1382 |
N->setDebugLoc(DebugLoc()); |
4 |
1382 |
N->setDebugLoc(DebugLoc()); |
0 |
| 1383 |
break; |
5 |
1383 |
break; |
5 |
| 1384 |
default: |
10 |
1384 |
default: |
10 |
| 1385 |
// When the node's point of use is located earlier in the instruction |
--- |
1385 |
// When the node's point of use is located earlier in the instruction |
--- |
| 1386 |
// sequence than its prior point of use, update its debug info to the |
--- |
1386 |
// sequence than its prior point of use, update its debug info to the |
--- |
| 1387 |
// earlier location. |
--- |
1387 |
// earlier location. |
--- |
| 1388 |
if (DL.getIROrder() && DL.getIROrder() < N->getIROrder()) |
10 |
1388 |
if (DL.getIROrder() && DL.getIROrder() < N->getIROrder()) |
10 |
| 1389 |
N->setDebugLoc(DL.getDebugLoc()); |
0 |
1389 |
N->setDebugLoc(DL.getDebugLoc()); |
0 |
| 1390 |
break; |
10 |
1390 |
break; |
10 |
| 1391 |
} |
--- |
1391 |
} |
--- |
| 1392 |
} |
--- |
1392 |
} |
--- |
| 1393 |
return N; |
71 |
1393 |
return N; |
71 |
| 1394 |
} |
--- |
1394 |
} |
--- |
| 1395 |
|
--- |
1395 |
|
--- |
| 1396 |
void SelectionDAG::clear() { |
4 |
1396 |
void SelectionDAG::clear() { |
4 |
| 1397 |
allnodes_clear(); |
4 |
1397 |
allnodes_clear(); |
4 |
| 1398 |
OperandRecycler.clear(OperandAllocator); |
4 |
1398 |
OperandRecycler.clear(OperandAllocator); |
4 |
| 1399 |
OperandAllocator.Reset(); |
4 |
1399 |
OperandAllocator.Reset(); |
4 |
| 1400 |
CSEMap.clear(); |
4 |
1400 |
CSEMap.clear(); |
4 |
| 1401 |
|
--- |
1401 |
|
--- |
| 1402 |
ExtendedValueTypeNodes.clear(); |
4 |
1402 |
ExtendedValueTypeNodes.clear(); |
4 |
| 1403 |
ExternalSymbols.clear(); |
4 |
1403 |
ExternalSymbols.clear(); |
4 |
| 1404 |
TargetExternalSymbols.clear(); |
4 |
1404 |
TargetExternalSymbols.clear(); |
4 |
| 1405 |
MCSymbols.clear(); |
4 |
1405 |
MCSymbols.clear(); |
4 |
| 1406 |
SDEI.clear(); |
4 |
1406 |
SDEI.clear(); |
4 |
| 1407 |
std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), |
4 |
1407 |
std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), |
4 |
| 1408 |
static_cast(nullptr)); |
--- |
1408 |
static_cast(nullptr)); |
--- |
| 1409 |
std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), |
4 |
1409 |
std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), |
4 |
| 1410 |
static_cast(nullptr)); |
--- |
1410 |
static_cast(nullptr)); |
--- |
| 1411 |
|
--- |
1411 |
|
--- |
| 1412 |
EntryNode.UseList = nullptr; |
4 |
1412 |
EntryNode.UseList = nullptr; |
4 |
| 1413 |
InsertNode(&EntryNode); |
4 |
1413 |
InsertNode(&EntryNode); |
4 |
| 1414 |
Root = getEntryNode(); |
4 |
1414 |
Root = getEntryNode(); |
4 |
| 1415 |
DbgInfo->clear(); |
4 |
1415 |
DbgInfo->clear(); |
4 |
| 1416 |
} |
4 |
1416 |
} |
4 |
| 1417 |
|
--- |
1417 |
|
--- |
| 1418 |
SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) { |
0 |
1418 |
SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) { |
0 |
| 1419 |
return VT.bitsGT(Op.getValueType()) |
0 |
1419 |
return VT.bitsGT(Op.getValueType()) |
0 |
| 1420 |
? getNode(ISD::FP_EXTEND, DL, VT, Op) |
0 |
1420 |
? getNode(ISD::FP_EXTEND, DL, VT, Op) |
0 |
| 1421 |
: getNode(ISD::FP_ROUND, DL, VT, Op, |
0 |
1421 |
: getNode(ISD::FP_ROUND, DL, VT, Op, |
0 |
| 1422 |
getIntPtrConstant(0, DL, /*isTarget=*/true)); |
0 |
1422 |
getIntPtrConstant(0, DL, /*isTarget=*/true)); |
0 |
| 1423 |
} |
--- |
1423 |
} |
--- |
| 1424 |
|
--- |
1424 |
|
--- |
| 1425 |
std::pair |
--- |
1425 |
std::pair |
--- |
| 1426 |
SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain, |
0 |
1426 |
SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain, |
0 |
| 1427 |
const SDLoc &DL, EVT VT) { |
--- |
1427 |
const SDLoc &DL, EVT VT) { |
--- |
| 1428 |
assert(!VT.bitsEq(Op.getValueType()) && |
0 |
1428 |
assert(!VT.bitsEq(Op.getValueType()) && |
0 |
| 1429 |
"Strict no-op FP extend/round not allowed."); |
--- |
1429 |
"Strict no-op FP extend/round not allowed."); |
--- |
| 1430 |
SDValue Res = |
--- |
1430 |
SDValue Res = |
--- |
| 1431 |
VT.bitsGT(Op.getValueType()) |
0 |
1431 |
VT.bitsGT(Op.getValueType()) |
0 |
| 1432 |
? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op}) |
0 |
1432 |
? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op}) |
0 |
| 1433 |
: getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other}, |
0 |
1433 |
: getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other}, |
0 |
| 1434 |
{Chain, Op, getIntPtrConstant(0, DL)}); |
0 |
1434 |
{Chain, Op, getIntPtrConstant(0, DL)}); |
0 |
| 1435 |
|
--- |
1435 |
|
--- |
| 1436 |
return std::pair(Res, SDValue(Res.getNode(), 1)); |
0 |
1436 |
return std::pair(Res, SDValue(Res.getNode(), 1)); |
0 |
| 1437 |
} |
--- |
1437 |
} |
--- |
| 1438 |
|
--- |
1438 |
|
--- |
| 1439 |
SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { |
0 |
1439 |
SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { |
0 |
| 1440 |
return VT.bitsGT(Op.getValueType()) ? |
0 |
1440 |
return VT.bitsGT(Op.getValueType()) ? |
0 |
| 1441 |
getNode(ISD::ANY_EXTEND, DL, VT, Op) : |
0 |
1441 |
getNode(ISD::ANY_EXTEND, DL, VT, Op) : |
0 |
| 1442 |
getNode(ISD::TRUNCATE, DL, VT, Op); |
0 |
1442 |
getNode(ISD::TRUNCATE, DL, VT, Op); |
0 |
| 1443 |
} |
--- |
1443 |
} |
--- |
| 1444 |
|
--- |
1444 |
|
--- |
| 1445 |
SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { |
1 |
1445 |
SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { |
1 |
| 1446 |
return VT.bitsGT(Op.getValueType()) ? |
1 |
1446 |
return VT.bitsGT(Op.getValueType()) ? |
1 |
| 1447 |
getNode(ISD::SIGN_EXTEND, DL, VT, Op) : |
0 |
1447 |
getNode(ISD::SIGN_EXTEND, DL, VT, Op) : |
0 |
| 1448 |
getNode(ISD::TRUNCATE, DL, VT, Op); |
1 |
1448 |
getNode(ISD::TRUNCATE, DL, VT, Op); |
1 |
| 1449 |
} |
--- |
1449 |
} |
--- |
| 1450 |
|
--- |
1450 |
|
--- |
| 1451 |
SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { |
0 |
1451 |
SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { |
0 |
| 1452 |
return VT.bitsGT(Op.getValueType()) ? |
0 |
1452 |
return VT.bitsGT(Op.getValueType()) ? |
0 |
| 1453 |
getNode(ISD::ZERO_EXTEND, DL, VT, Op) : |
0 |
1453 |
getNode(ISD::ZERO_EXTEND, DL, VT, Op) : |
0 |
| 1454 |
getNode(ISD::TRUNCATE, DL, VT, Op); |
0 |
1454 |
getNode(ISD::TRUNCATE, DL, VT, Op); |
0 |
| 1455 |
} |
--- |
1455 |
} |
--- |
| 1456 |
|
--- |
1456 |
|
--- |
| 1457 |
SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, |
0 |
1457 |
SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, |
0 |
| 1458 |
EVT OpVT) { |
--- |
1458 |
EVT OpVT) { |
--- |
| 1459 |
if (VT.bitsLE(Op.getValueType())) |
0 |
1459 |
if (VT.bitsLE(Op.getValueType())) |
0 |
| 1460 |
return getNode(ISD::TRUNCATE, SL, VT, Op); |
0 |
1460 |
return getNode(ISD::TRUNCATE, SL, VT, Op); |
0 |
| 1461 |
|
--- |
1461 |
|
--- |
| 1462 |
TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT); |
0 |
1462 |
TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT); |
0 |
| 1463 |
return getNode(TLI->getExtendForContent(BType), SL, VT, Op); |
0 |
1463 |
return getNode(TLI->getExtendForContent(BType), SL, VT, Op); |
0 |
| 1464 |
} |
--- |
1464 |
} |
--- |
| 1465 |
|
--- |
1465 |
|
--- |
| 1466 |
SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) { |
1 |
1466 |
SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) { |
1 |
| 1467 |
EVT OpVT = Op.getValueType(); |
1 |
1467 |
EVT OpVT = Op.getValueType(); |
1 |
| 1468 |
assert(VT.isInteger() && OpVT.isInteger() && |
1 |
1468 |
assert(VT.isInteger() && OpVT.isInteger() && |
1 |
| 1469 |
"Cannot getZeroExtendInReg FP types"); |
--- |
1469 |
"Cannot getZeroExtendInReg FP types"); |
--- |
| 1470 |
assert(VT.isVector() == OpVT.isVector() && |
1 |
1470 |
assert(VT.isVector() == OpVT.isVector() && |
1 |
| 1471 |
"getZeroExtendInReg type should be vector iff the operand " |
--- |
1471 |
"getZeroExtendInReg type should be vector iff the operand " |
--- |
| 1472 |
"type is vector!"); |
--- |
1472 |
"type is vector!"); |
--- |
| 1473 |
assert((!VT.isVector() || |
1 |
1473 |
assert((!VT.isVector() || |
1 |
| 1474 |
VT.getVectorElementCount() == OpVT.getVectorElementCount()) && |
--- |
1474 |
VT.getVectorElementCount() == OpVT.getVectorElementCount()) && |
--- |
| 1475 |
"Vector element counts must match in getZeroExtendInReg"); |
--- |
1475 |
"Vector element counts must match in getZeroExtendInReg"); |
--- |
| 1476 |
assert(VT.bitsLE(OpVT) && "Not extending!"); |
1 |
1476 |
assert(VT.bitsLE(OpVT) && "Not extending!"); |
1 |
| 1477 |
if (OpVT == VT) |
1 |
1477 |
if (OpVT == VT) |
1 |
| 1478 |
return Op; |
0 |
1478 |
return Op; |
0 |
| 1479 |
APInt Imm = APInt::getLowBitsSet(OpVT.getScalarSizeInBits(), |
1 |
1479 |
APInt Imm = APInt::getLowBitsSet(OpVT.getScalarSizeInBits(), |
1 |
| 1480 |
VT.getScalarSizeInBits()); |
1 |
1480 |
VT.getScalarSizeInBits()); |
1 |
| 1481 |
return getNode(ISD::AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT)); |
1 |
1481 |
return getNode(ISD::AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT)); |
1 |
| 1482 |
} |
1 |
1482 |
} |
1 |
| 1483 |
|
--- |
1483 |
|
--- |
| 1484 |
SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { |
0 |
1484 |
SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { |
0 |
| 1485 |
// Only unsigned pointer semantics are supported right now. In the future this |
--- |
1485 |
// Only unsigned pointer semantics are supported right now. In the future this |
--- |
| 1486 |
// might delegate to TLI to check pointer signedness. |
--- |
1486 |
// might delegate to TLI to check pointer signedness. |
--- |
| 1487 |
return getZExtOrTrunc(Op, DL, VT); |
0 |
1487 |
return getZExtOrTrunc(Op, DL, VT); |
0 |
| 1488 |
} |
--- |
1488 |
} |
--- |
| 1489 |
|
--- |
1489 |
|
--- |
| 1490 |
SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) { |
0 |
1490 |
SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) { |
0 |
| 1491 |
// Only unsigned pointer semantics are supported right now. In the future this |
--- |
1491 |
// Only unsigned pointer semantics are supported right now. In the future this |
--- |
| 1492 |
// might delegate to TLI to check pointer signedness. |
--- |
1492 |
// might delegate to TLI to check pointer signedness. |
--- |
| 1493 |
return getZeroExtendInReg(Op, DL, VT); |
0 |
1493 |
return getZeroExtendInReg(Op, DL, VT); |
0 |
| 1494 |
} |
--- |
1494 |
} |
--- |
| 1495 |
|
--- |
1495 |
|
--- |
| 1496 |
SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) { |
0 |
1496 |
SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) { |
0 |
| 1497 |
return getNode(ISD::SUB, DL, VT, getConstant(0, DL, VT), Val); |
0 |
1497 |
return getNode(ISD::SUB, DL, VT, getConstant(0, DL, VT), Val); |
0 |
| 1498 |
} |
--- |
1498 |
} |
--- |
| 1499 |
|
--- |
1499 |
|
--- |
| 1500 |
/// getNOT - Create a bitwise NOT operation as (XOR Val, -1). |
--- |
1500 |
/// getNOT - Create a bitwise NOT operation as (XOR Val, -1). |
--- |
| 1501 |
SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) { |
0 |
1501 |
SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) { |
0 |
| 1502 |
return getNode(ISD::XOR, DL, VT, Val, getAllOnesConstant(DL, VT)); |
0 |
1502 |
return getNode(ISD::XOR, DL, VT, Val, getAllOnesConstant(DL, VT)); |
0 |
| 1503 |
} |
--- |
1503 |
} |
--- |
| 1504 |
|
--- |
1504 |
|
--- |
| 1505 |
SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) { |
0 |
1505 |
SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) { |
0 |
| 1506 |
SDValue TrueValue = getBoolConstant(true, DL, VT, VT); |
0 |
1506 |
SDValue TrueValue = getBoolConstant(true, DL, VT, VT); |
0 |
| 1507 |
return getNode(ISD::XOR, DL, VT, Val, TrueValue); |
0 |
1507 |
return getNode(ISD::XOR, DL, VT, Val, TrueValue); |
0 |
| 1508 |
} |
--- |
1508 |
} |
--- |
| 1509 |
|
--- |
1509 |
|
--- |
| 1510 |
SDValue SelectionDAG::getVPLogicalNOT(const SDLoc &DL, SDValue Val, |
0 |
1510 |
SDValue SelectionDAG::getVPLogicalNOT(const SDLoc &DL, SDValue Val, |
0 |
| 1511 |
SDValue Mask, SDValue EVL, EVT VT) { |
--- |
1511 |
SDValue Mask, SDValue EVL, EVT VT) { |
--- |
| 1512 |
SDValue TrueValue = getBoolConstant(true, DL, VT, VT); |
0 |
1512 |
SDValue TrueValue = getBoolConstant(true, DL, VT, VT); |
0 |
| 1513 |
return getNode(ISD::VP_XOR, DL, VT, Val, TrueValue, Mask, EVL); |
0 |
1513 |
return getNode(ISD::VP_XOR, DL, VT, Val, TrueValue, Mask, EVL); |
0 |
| 1514 |
} |
--- |
1514 |
} |
--- |
| 1515 |
|
--- |
1515 |
|
--- |
| 1516 |
SDValue SelectionDAG::getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, |
0 |
1516 |
SDValue SelectionDAG::getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, |
0 |
| 1517 |
SDValue Mask, SDValue EVL) { |
--- |
1517 |
SDValue Mask, SDValue EVL) { |
--- |
| 1518 |
return getVPZExtOrTrunc(DL, VT, Op, Mask, EVL); |
0 |
1518 |
return getVPZExtOrTrunc(DL, VT, Op, Mask, EVL); |
0 |
| 1519 |
} |
--- |
1519 |
} |
--- |
| 1520 |
|
--- |
1520 |
|
--- |
| 1521 |
SDValue SelectionDAG::getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, |
0 |
1521 |
SDValue SelectionDAG::getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, |
0 |
| 1522 |
SDValue Mask, SDValue EVL) { |
--- |
1522 |
SDValue Mask, SDValue EVL) { |
--- |
| 1523 |
if (VT.bitsGT(Op.getValueType())) |
0 |
1523 |
if (VT.bitsGT(Op.getValueType())) |
0 |
| 1524 |
return getNode(ISD::VP_ZERO_EXTEND, DL, VT, Op, Mask, EVL); |
0 |
1524 |
return getNode(ISD::VP_ZERO_EXTEND, DL, VT, Op, Mask, EVL); |
0 |
| 1525 |
if (VT.bitsLT(Op.getValueType())) |
0 |
1525 |
if (VT.bitsLT(Op.getValueType())) |
0 |
| 1526 |
return getNode(ISD::VP_TRUNCATE, DL, VT, Op, Mask, EVL); |
0 |
1526 |
return getNode(ISD::VP_TRUNCATE, DL, VT, Op, Mask, EVL); |
0 |
| 1527 |
return Op; |
0 |
1527 |
return Op; |
0 |
| 1528 |
} |
--- |
1528 |
} |
--- |
| 1529 |
|
--- |
1529 |
|
--- |
| 1530 |
SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT, |
0 |
1530 |
SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT, |
0 |
| 1531 |
EVT OpVT) { |
--- |
1531 |
EVT OpVT) { |
--- |
| 1532 |
if (!V) |
0 |
1532 |
if (!V) |
0 |
| 1533 |
return getConstant(0, DL, VT); |
0 |
1533 |
return getConstant(0, DL, VT); |
0 |
| 1534 |
|
--- |
1534 |
|
--- |
| 1535 |
switch (TLI->getBooleanContents(OpVT)) { |
0 |
1535 |
switch (TLI->getBooleanContents(OpVT)) { |
0 |
| 1536 |
case TargetLowering::ZeroOrOneBooleanContent: |
0 |
1536 |
case TargetLowering::ZeroOrOneBooleanContent: |
0 |
| 1537 |
case TargetLowering::UndefinedBooleanContent: |
--- |
1537 |
case TargetLowering::UndefinedBooleanContent: |
--- |
| 1538 |
return getConstant(1, DL, VT); |
0 |
1538 |
return getConstant(1, DL, VT); |
0 |
| 1539 |
case TargetLowering::ZeroOrNegativeOneBooleanContent: |
0 |
1539 |
case TargetLowering::ZeroOrNegativeOneBooleanContent: |
0 |
| 1540 |
return getAllOnesConstant(DL, VT); |
0 |
1540 |
return getAllOnesConstant(DL, VT); |
0 |
| 1541 |
} |
--- |
1541 |
} |
--- |
| 1542 |
llvm_unreachable("Unexpected boolean content enum!"); |
0 |
1542 |
llvm_unreachable("Unexpected boolean content enum!"); |
0 |
| 1543 |
} |
--- |
1543 |
} |
--- |
| 1544 |
|
--- |
1544 |
|
--- |
| 1545 |
SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT, |
19 |
1545 |
SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT, |
19 |
| 1546 |
bool isT, bool isO) { |
--- |
1546 |
bool isT, bool isO) { |
--- |
| 1547 |
EVT EltVT = VT.getScalarType(); |
19 |
1547 |
EVT EltVT = VT.getScalarType(); |
19 |
| 1548 |
assert((EltVT.getSizeInBits() >= 64 || |
19 |
1548 |
assert((EltVT.getSizeInBits() >= 64 || |
19 |
| 1549 |
(uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && |
--- |
1549 |
(uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && |
--- |
| 1550 |
"getConstant with a uint64_t value that doesn't fit in the type!"); |
--- |
1550 |
"getConstant with a uint64_t value that doesn't fit in the type!"); |
--- |
| 1551 |
return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO); |
19 |
1551 |
return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO); |
19 |
| 1552 |
} |
--- |
1552 |
} |
--- |
| 1553 |
|
--- |
1553 |
|
--- |
| 1554 |
SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT, |
20 |
1554 |
SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT, |
20 |
| 1555 |
bool isT, bool isO) { |
--- |
1555 |
bool isT, bool isO) { |
--- |
| 1556 |
return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO); |
20 |
1556 |
return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO); |
20 |
| 1557 |
} |
--- |
1557 |
} |
--- |
| 1558 |
|
--- |
1558 |
|
--- |
| 1559 |
SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL, |
20 |
1559 |
SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL, |
20 |
| 1560 |
EVT VT, bool isT, bool isO) { |
--- |
1560 |
EVT VT, bool isT, bool isO) { |
--- |
| 1561 |
assert(VT.isInteger() && "Cannot create FP integer constant!"); |
20 |
1561 |
assert(VT.isInteger() && "Cannot create FP integer constant!"); |
20 |
| 1562 |
|
--- |
1562 |
|
--- |
| 1563 |
EVT EltVT = VT.getScalarType(); |
20 |
1563 |
EVT EltVT = VT.getScalarType(); |
20 |
| 1564 |
const ConstantInt *Elt = &Val; |
20 |
1564 |
const ConstantInt *Elt = &Val; |
20 |
| 1565 |
|
--- |
1565 |
|
--- |
| 1566 |
// In some cases the vector type is legal but the element type is illegal and |
--- |
1566 |
// In some cases the vector type is legal but the element type is illegal and |
--- |
| 1567 |
// needs to be promoted, for example v8i8 on ARM. In this case, promote the |
--- |
1567 |
// needs to be promoted, for example v8i8 on ARM. In this case, promote the |
--- |
| 1568 |
// inserted value (the type does not need to match the vector element type). |
--- |
1568 |
// inserted value (the type does not need to match the vector element type). |
--- |
| 1569 |
// Any extra bits introduced will be truncated away. |
--- |
1569 |
// Any extra bits introduced will be truncated away. |
--- |
| 1570 |
if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) == |
20 |
1570 |
if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) == |
20 |
| 1571 |
TargetLowering::TypePromoteInteger) { |
--- |
1571 |
TargetLowering::TypePromoteInteger) { |
--- |
| 1572 |
EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); |
0 |
1572 |
EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); |
0 |
| 1573 |
APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits()); |
0 |
1573 |
APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits()); |
0 |
| 1574 |
Elt = ConstantInt::get(*getContext(), NewVal); |
0 |
1574 |
Elt = ConstantInt::get(*getContext(), NewVal); |
0 |
| 1575 |
} |
0 |
1575 |
} |
0 |
| 1576 |
// In other cases the element type is illegal and needs to be expanded, for |
--- |
1576 |
// In other cases the element type is illegal and needs to be expanded, for |
--- |
| 1577 |
// example v2i64 on MIPS32. In this case, find the nearest legal type, split |
--- |
1577 |
// example v2i64 on MIPS32. In this case, find the nearest legal type, split |
--- |
| 1578 |
// the value into n parts and use a vector type with n-times the elements. |
--- |
1578 |
// the value into n parts and use a vector type with n-times the elements. |
--- |
| 1579 |
// Then bitcast to the type requested. |
--- |
1579 |
// Then bitcast to the type requested. |
--- |
| 1580 |
// Legalizing constants too early makes the DAGCombiner's job harder so we |
--- |
1580 |
// Legalizing constants too early makes the DAGCombiner's job harder so we |
--- |
| 1581 |
// only legalize if the DAG tells us we must produce legal types. |
--- |
1581 |
// only legalize if the DAG tells us we must produce legal types. |
--- |
| 1582 |
else if (NewNodesMustHaveLegalTypes && VT.isVector() && |
20 |
1582 |
else if (NewNodesMustHaveLegalTypes && VT.isVector() && |
20 |
| 1583 |
TLI->getTypeAction(*getContext(), EltVT) == |
0 |
1583 |
TLI->getTypeAction(*getContext(), EltVT) == |
0 |
| 1584 |
TargetLowering::TypeExpandInteger) { |
--- |
1584 |
TargetLowering::TypeExpandInteger) { |
--- |
| 1585 |
const APInt &NewVal = Elt->getValue(); |
0 |
1585 |
const APInt &NewVal = Elt->getValue(); |
0 |
| 1586 |
EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); |
0 |
1586 |
EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); |
0 |
| 1587 |
unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits(); |
0 |
1587 |
unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits(); |
0 |
| 1588 |
|
--- |
1588 |
|
--- |
| 1589 |
// For scalable vectors, try to use a SPLAT_VECTOR_PARTS node. |
--- |
1589 |
// For scalable vectors, try to use a SPLAT_VECTOR_PARTS node. |
--- |
| 1590 |
if (VT.isScalableVector()) { |
0 |
1590 |
if (VT.isScalableVector()) { |
0 |
| 1591 |
assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 && |
0 |
1591 |
assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 && |
0 |
| 1592 |
"Can only handle an even split!"); |
--- |
1592 |
"Can only handle an even split!"); |
--- |
| 1593 |
unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits; |
0 |
1593 |
unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits; |
0 |
| 1594 |
|
--- |
1594 |
|
--- |
| 1595 |
SmallVector ScalarParts; |
0 |
1595 |
SmallVector ScalarParts; |
0 |
| 1596 |
for (unsigned i = 0; i != Parts; ++i) |
0 |
1596 |
for (unsigned i = 0; i != Parts; ++i) |
0 |
| 1597 |
ScalarParts.push_back(getConstant( |
0 |
1597 |
ScalarParts.push_back(getConstant( |
0 |
| 1598 |
NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL, |
0 |
1598 |
NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL, |
0 |
| 1599 |
ViaEltVT, isT, isO)); |
--- |
1599 |
ViaEltVT, isT, isO)); |
--- |
| 1600 |
|
--- |
1600 |
|
--- |
| 1601 |
return getNode(ISD::SPLAT_VECTOR_PARTS, DL, VT, ScalarParts); |
0 |
1601 |
return getNode(ISD::SPLAT_VECTOR_PARTS, DL, VT, ScalarParts); |
0 |
| 1602 |
} |
0 |
1602 |
} |
0 |
| 1603 |
|
--- |
1603 |
|
--- |
| 1604 |
unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits; |
0 |
1604 |
unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits; |
0 |
| 1605 |
EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts); |
0 |
1605 |
EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts); |
0 |
| 1606 |
|
--- |
1606 |
|
--- |
| 1607 |
// Check the temporary vector is the correct size. If this fails then |
--- |
1607 |
// Check the temporary vector is the correct size. If this fails then |
--- |
| 1608 |
// getTypeToTransformTo() probably returned a type whose size (in bits) |
--- |
1608 |
// getTypeToTransformTo() probably returned a type whose size (in bits) |
--- |
| 1609 |
// isn't a power-of-2 factor of the requested type size. |
--- |
1609 |
// isn't a power-of-2 factor of the requested type size. |
--- |
| 1610 |
assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits()); |
0 |
1610 |
assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits()); |
0 |
| 1611 |
|
--- |
1611 |
|
--- |
| 1612 |
SmallVector EltParts; |
0 |
1612 |
SmallVector EltParts; |
0 |
| 1613 |
for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) |
0 |
1613 |
for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) |
0 |
| 1614 |
EltParts.push_back(getConstant( |
0 |
1614 |
EltParts.push_back(getConstant( |
0 |
| 1615 |
NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL, |
0 |
1615 |
NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL, |
0 |
| 1616 |
ViaEltVT, isT, isO)); |
--- |
1616 |
ViaEltVT, isT, isO)); |
--- |
| 1617 |
|
--- |
1617 |
|
--- |
| 1618 |
// EltParts is currently in little endian order. If we actually want |
--- |
1618 |
// EltParts is currently in little endian order. If we actually want |
--- |
| 1619 |
// big-endian order then reverse it now. |
--- |
1619 |
// big-endian order then reverse it now. |
--- |
| 1620 |
if (getDataLayout().isBigEndian()) |
0 |
1620 |
if (getDataLayout().isBigEndian()) |
0 |
| 1621 |
std::reverse(EltParts.begin(), EltParts.end()); |
0 |
1621 |
std::reverse(EltParts.begin(), EltParts.end()); |
0 |
| 1622 |
|
--- |
1622 |
|
--- |
| 1623 |
// The elements must be reversed when the element order is different |
--- |
1623 |
// The elements must be reversed when the element order is different |
--- |
| 1624 |
// to the endianness of the elements (because the BITCAST is itself a |
--- |
1624 |
// to the endianness of the elements (because the BITCAST is itself a |
--- |
| 1625 |
// vector shuffle in this situation). However, we do not need any code to |
--- |
1625 |
// vector shuffle in this situation). However, we do not need any code to |
--- |
| 1626 |
// perform this reversal because getConstant() is producing a vector |
--- |
1626 |
// perform this reversal because getConstant() is producing a vector |
--- |
| 1627 |
// splat. |
--- |
1627 |
// splat. |
--- |
| 1628 |
// This situation occurs in MIPS MSA. |
--- |
1628 |
// This situation occurs in MIPS MSA. |
--- |
| 1629 |
|
--- |
1629 |
|
--- |
| 1630 |
SmallVector Ops; |
0 |
1630 |
SmallVector Ops; |
0 |
| 1631 |
for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) |
0 |
1631 |
for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) |
0 |
| 1632 |
llvm::append_range(Ops, EltParts); |
0 |
1632 |
llvm::append_range(Ops, EltParts); |
0 |
| 1633 |
|
--- |
1633 |
|
--- |
| 1634 |
SDValue V = |
--- |
1634 |
SDValue V = |
--- |
| 1635 |
getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops)); |
0 |
1635 |
getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops)); |
0 |
| 1636 |
return V; |
0 |
1636 |
return V; |
0 |
| 1637 |
} |
0 |
1637 |
} |
0 |
| 1638 |
|
--- |
1638 |
|
--- |
| 1639 |
assert(Elt->getBitWidth() == EltVT.getSizeInBits() && |
20 |
1639 |
assert(Elt->getBitWidth() == EltVT.getSizeInBits() && |
20 |
| 1640 |
"APInt size does not match type size!"); |
--- |
1640 |
"APInt size does not match type size!"); |
--- |
| 1641 |
unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; |
20 |
1641 |
unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; |
20 |
| 1642 |
FoldingSetNodeID ID; |
20 |
1642 |
FoldingSetNodeID ID; |
20 |
| 1643 |
AddNodeIDNode(ID, Opc, getVTList(EltVT), std::nullopt); |
20 |
1643 |
AddNodeIDNode(ID, Opc, getVTList(EltVT), std::nullopt); |
20 |
| 1644 |
ID.AddPointer(Elt); |
20 |
1644 |
ID.AddPointer(Elt); |
20 |
| 1645 |
ID.AddBoolean(isO); |
20 |
1645 |
ID.AddBoolean(isO); |
20 |
| 1646 |
void *IP = nullptr; |
20 |
1646 |
void *IP = nullptr; |
20 |
| 1647 |
SDNode *N = nullptr; |
20 |
1647 |
SDNode *N = nullptr; |
20 |
| 1648 |
if ((N = FindNodeOrInsertPos(ID, DL, IP))) |
20 |
1648 |
if ((N = FindNodeOrInsertPos(ID, DL, IP))) |
20 |
| 1649 |
if (!VT.isVector()) |
10 |
1649 |
if (!VT.isVector()) |
10 |
| 1650 |
return SDValue(N, 0); |
10 |
1650 |
return SDValue(N, 0); |
10 |
| 1651 |
|
--- |
1651 |
|
--- |
| 1652 |
if (!N) { |
10 |
1652 |
if (!N) { |
10 |
| 1653 |
N = newSDNode(isT, isO, Elt, EltVT); |
10 |
1653 |
N = newSDNode(isT, isO, Elt, EltVT); |
10 |
| 1654 |
CSEMap.InsertNode(N, IP); |
10 |
1654 |
CSEMap.InsertNode(N, IP); |
10 |
| 1655 |
InsertNode(N); |
10 |
1655 |
InsertNode(N); |
10 |
| 1656 |
NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this); |
10 |
1656 |
NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this); |
10 |
| 1657 |
} |
--- |
1657 |
} |
--- |
| 1658 |
|
--- |
1658 |
|
--- |
| 1659 |
SDValue Result(N, 0); |
10 |
1659 |
SDValue Result(N, 0); |
10 |
| 1660 |
if (VT.isVector()) |
10 |
1660 |
if (VT.isVector()) |
10 |
| 1661 |
Result = getSplat(VT, DL, Result); |
0 |
1661 |
Result = getSplat(VT, DL, Result); |
0 |
| 1662 |
return Result; |
10 |
1662 |
return Result; |
10 |
| 1663 |
} |
20 |
1663 |
} |
20 |
| 1664 |
|
--- |
1664 |
|
--- |
| 1665 |
SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL, |
0 |
1665 |
SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL, |
0 |
| 1666 |
bool isTarget) { |
--- |
1666 |
bool isTarget) { |
--- |
| 1667 |
return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget); |
0 |
1667 |
return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget); |
0 |
| 1668 |
} |
--- |
1668 |
} |
--- |
| 1669 |
|
--- |
1669 |
|
--- |
| 1670 |
SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT, |
0 |
1670 |
SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT, |
0 |
| 1671 |
const SDLoc &DL, bool LegalTypes) { |
--- |
1671 |
const SDLoc &DL, bool LegalTypes) { |
--- |
| 1672 |
assert(VT.isInteger() && "Shift amount is not an integer type!"); |
0 |
1672 |
assert(VT.isInteger() && "Shift amount is not an integer type!"); |
0 |
| 1673 |
EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout(), LegalTypes); |
0 |
1673 |
EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout(), LegalTypes); |
0 |
| 1674 |
return getConstant(Val, DL, ShiftVT); |
0 |
1674 |
return getConstant(Val, DL, ShiftVT); |
0 |
| 1675 |
} |
--- |
1675 |
} |
--- |
| 1676 |
|
--- |
1676 |
|
--- |
| 1677 |
SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL, |
0 |
1677 |
SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL, |
0 |
| 1678 |
bool isTarget) { |
--- |
1678 |
bool isTarget) { |
--- |
| 1679 |
return getConstant(Val, DL, TLI->getVectorIdxTy(getDataLayout()), isTarget); |
0 |
1679 |
return getConstant(Val, DL, TLI->getVectorIdxTy(getDataLayout()), isTarget); |
0 |
| 1680 |
} |
--- |
1680 |
} |
--- |
| 1681 |
|
--- |
1681 |
|
--- |
| 1682 |
SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT, |
0 |
1682 |
SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT, |
0 |
| 1683 |
bool isTarget) { |
--- |
1683 |
bool isTarget) { |
--- |
| 1684 |
return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget); |
0 |
1684 |
return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget); |
0 |
| 1685 |
} |
--- |
1685 |
} |
--- |
| 1686 |
|
--- |
1686 |
|
--- |
| 1687 |
SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL, |
0 |
1687 |
SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL, |
0 |
| 1688 |
EVT VT, bool isTarget) { |
--- |
1688 |
EVT VT, bool isTarget) { |
--- |
| 1689 |
assert(VT.isFloatingPoint() && "Cannot create integer FP constant!"); |
0 |
1689 |
assert(VT.isFloatingPoint() && "Cannot create integer FP constant!"); |
0 |
| 1690 |
|
--- |
1690 |
|
--- |
| 1691 |
EVT EltVT = VT.getScalarType(); |
0 |
1691 |
EVT EltVT = VT.getScalarType(); |
0 |
| 1692 |
|
--- |
1692 |
|
--- |
| 1693 |
// Do the map lookup using the actual bit pattern for the floating point |
--- |
1693 |
// Do the map lookup using the actual bit pattern for the floating point |
--- |
| 1694 |
// value, so that we don't have problems with 0.0 comparing equal to -0.0, and |
--- |
1694 |
// value, so that we don't have problems with 0.0 comparing equal to -0.0, and |
--- |
| 1695 |
// we don't have issues with SNANs. |
--- |
1695 |
// we don't have issues with SNANs. |
--- |
| 1696 |
unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; |
0 |
1696 |
unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; |
0 |
| 1697 |
FoldingSetNodeID ID; |
0 |
1697 |
FoldingSetNodeID ID; |
0 |
| 1698 |
AddNodeIDNode(ID, Opc, getVTList(EltVT), std::nullopt); |
0 |
1698 |
AddNodeIDNode(ID, Opc, getVTList(EltVT), std::nullopt); |
0 |
| 1699 |
ID.AddPointer(&V); |
0 |
1699 |
ID.AddPointer(&V); |
0 |
| 1700 |
void *IP = nullptr; |
0 |
1700 |
void *IP = nullptr; |
0 |
| 1701 |
SDNode *N = nullptr; |
0 |
1701 |
SDNode *N = nullptr; |
0 |
| 1702 |
if ((N = FindNodeOrInsertPos(ID, DL, IP))) |
0 |
1702 |
if ((N = FindNodeOrInsertPos(ID, DL, IP))) |
0 |
| 1703 |
if (!VT.isVector()) |
0 |
1703 |
if (!VT.isVector()) |
0 |
| 1704 |
return SDValue(N, 0); |
0 |
1704 |
return SDValue(N, 0); |
0 |
| 1705 |
|
--- |
1705 |
|
--- |
| 1706 |
if (!N) { |
0 |
1706 |
if (!N) { |
0 |
| 1707 |
N = newSDNode(isTarget, &V, EltVT); |
0 |
1707 |
N = newSDNode(isTarget, &V, EltVT); |
0 |
| 1708 |
CSEMap.InsertNode(N, IP); |
0 |
1708 |
CSEMap.InsertNode(N, IP); |
0 |
| 1709 |
InsertNode(N); |
0 |
1709 |
InsertNode(N); |
0 |
| 1710 |
} |
--- |
1710 |
} |
--- |
| 1711 |
|
--- |
1711 |
|
--- |
| 1712 |
SDValue Result(N, 0); |
0 |
1712 |
SDValue Result(N, 0); |
0 |
| 1713 |
if (VT.isVector()) |
0 |
1713 |
if (VT.isVector()) |
0 |
| 1714 |
Result = getSplat(VT, DL, Result); |
0 |
1714 |
Result = getSplat(VT, DL, Result); |
0 |
| 1715 |
NewSDValueDbgMsg(Result, "Creating fp constant: ", this); |
0 |
1715 |
NewSDValueDbgMsg(Result, "Creating fp constant: ", this); |
0 |
| 1716 |
return Result; |
0 |
1716 |
return Result; |
0 |
| 1717 |
} |
0 |
1717 |
} |
0 |
| 1718 |
|
--- |
1718 |
|
--- |
| 1719 |
SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT, |
0 |
1719 |
SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT, |
0 |
| 1720 |
bool isTarget) { |
--- |
1720 |
bool isTarget) { |
--- |
| 1721 |
EVT EltVT = VT.getScalarType(); |
0 |
1721 |
EVT EltVT = VT.getScalarType(); |
0 |
| 1722 |
if (EltVT == MVT::f32) |
0 |
1722 |
if (EltVT == MVT::f32) |
0 |
| 1723 |
return getConstantFP(APFloat((float)Val), DL, VT, isTarget); |
0 |
1723 |
return getConstantFP(APFloat((float)Val), DL, VT, isTarget); |
0 |
| 1724 |
if (EltVT == MVT::f64) |
0 |
1724 |
if (EltVT == MVT::f64) |
0 |
| 1725 |
return getConstantFP(APFloat(Val), DL, VT, isTarget); |
0 |
1725 |
return getConstantFP(APFloat(Val), DL, VT, isTarget); |
0 |
| 1726 |
if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 || |
0 |
1726 |
if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 || |
0 |
| 1727 |
EltVT == MVT::f16 || EltVT == MVT::bf16) { |
0 |
1727 |
EltVT == MVT::f16 || EltVT == MVT::bf16) { |
0 |
| 1728 |
bool Ignored; |
--- |
1728 |
bool Ignored; |
--- |
| 1729 |
APFloat APF = APFloat(Val); |
0 |
1729 |
APFloat APF = APFloat(Val); |
0 |
| 1730 |
APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, |
0 |
1730 |
APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, |
0 |
| 1731 |
&Ignored); |
--- |
1731 |
&Ignored); |
--- |
| 1732 |
return getConstantFP(APF, DL, VT, isTarget); |
0 |
1732 |
return getConstantFP(APF, DL, VT, isTarget); |
0 |
| 1733 |
} |
0 |
1733 |
} |
0 |
| 1734 |
llvm_unreachable("Unsupported type in getConstantFP"); |
0 |
1734 |
llvm_unreachable("Unsupported type in getConstantFP"); |
0 |
| 1735 |
} |
--- |
1735 |
} |
--- |
| 1736 |
|
--- |
1736 |
|
--- |
| 1737 |
SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, |
0 |
1737 |
SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, |
0 |
| 1738 |
EVT VT, int64_t Offset, bool isTargetGA, |
--- |
1738 |
EVT VT, int64_t Offset, bool isTargetGA, |
--- |
| 1739 |
unsigned TargetFlags) { |
--- |
1739 |
unsigned TargetFlags) { |
--- |
| 1740 |
assert((TargetFlags == 0 || isTargetGA) && |
0 |
1740 |
assert((TargetFlags == 0 || isTargetGA) && |
0 |
| 1741 |
"Cannot set target flags on target-independent globals"); |
--- |
1741 |
"Cannot set target flags on target-independent globals"); |
--- |
| 1742 |
|
--- |
1742 |
|
--- |
| 1743 |
// Truncate (with sign-extension) the offset value to the pointer size. |
--- |
1743 |
// Truncate (with sign-extension) the offset value to the pointer size. |
--- |
| 1744 |
unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType()); |
0 |
1744 |
unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType()); |
0 |
| 1745 |
if (BitWidth < 64) |
0 |
1745 |
if (BitWidth < 64) |
0 |
| 1746 |
Offset = SignExtend64(Offset, BitWidth); |
0 |
1746 |
Offset = SignExtend64(Offset, BitWidth); |
0 |
| 1747 |
|
--- |
1747 |
|
--- |
| 1748 |
unsigned Opc; |
--- |
1748 |
unsigned Opc; |
--- |
| 1749 |
if (GV->isThreadLocal()) |
0 |
1749 |
if (GV->isThreadLocal()) |
0 |
| 1750 |
Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; |
0 |
1750 |
Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; |
0 |
| 1751 |
else |
--- |
1751 |
else |
--- |
| 1752 |
Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; |
0 |
1752 |
Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; |
0 |
| 1753 |
|
--- |
1753 |
|
--- |
| 1754 |
FoldingSetNodeID ID; |
0 |
1754 |
FoldingSetNodeID ID; |
0 |
| 1755 |
AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt); |
0 |
1755 |
AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt); |
0 |
| 1756 |
ID.AddPointer(GV); |
0 |
1756 |
ID.AddPointer(GV); |
0 |
| 1757 |
ID.AddInteger(Offset); |
0 |
1757 |
ID.AddInteger(Offset); |
0 |
| 1758 |
ID.AddInteger(TargetFlags); |
0 |
1758 |
ID.AddInteger(TargetFlags); |
0 |
| 1759 |
void *IP = nullptr; |
0 |
1759 |
void *IP = nullptr; |
0 |
| 1760 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) |
0 |
1760 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) |
0 |
| 1761 |
return SDValue(E, 0); |
0 |
1761 |
return SDValue(E, 0); |
0 |
| 1762 |
|
--- |
1762 |
|
--- |
| 1763 |
auto *N = newSDNode( |
0 |
1763 |
auto *N = newSDNode( |
0 |
| 1764 |
Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags); |
0 |
1764 |
Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags); |
0 |
| 1765 |
CSEMap.InsertNode(N, IP); |
0 |
1765 |
CSEMap.InsertNode(N, IP); |
0 |
| 1766 |
InsertNode(N); |
0 |
1766 |
InsertNode(N); |
0 |
| 1767 |
return SDValue(N, 0); |
0 |
1767 |
return SDValue(N, 0); |
0 |
| 1768 |
} |
0 |
1768 |
} |
0 |
| 1769 |
|
--- |
1769 |
|
--- |
| 1770 |
SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { |
16 |
1770 |
SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { |
16 |
| 1771 |
unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; |
16 |
1771 |
unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; |
16 |
| 1772 |
FoldingSetNodeID ID; |
16 |
1772 |
FoldingSetNodeID ID; |
16 |
| 1773 |
AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt); |
16 |
1773 |
AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt); |
16 |
| 1774 |
ID.AddInteger(FI); |
16 |
1774 |
ID.AddInteger(FI); |
16 |
| 1775 |
void *IP = nullptr; |
16 |
1775 |
void *IP = nullptr; |
16 |
| 1776 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
16 |
1776 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
16 |
| 1777 |
return SDValue(E, 0); |
2 |
1777 |
return SDValue(E, 0); |
2 |
| 1778 |
|
--- |
1778 |
|
--- |
| 1779 |
auto *N = newSDNode(FI, VT, isTarget); |
14 |
1779 |
auto *N = newSDNode(FI, VT, isTarget); |
14 |
| 1780 |
CSEMap.InsertNode(N, IP); |
14 |
1780 |
CSEMap.InsertNode(N, IP); |
14 |
| 1781 |
InsertNode(N); |
14 |
1781 |
InsertNode(N); |
14 |
| 1782 |
return SDValue(N, 0); |
14 |
1782 |
return SDValue(N, 0); |
14 |
| 1783 |
} |
16 |
1783 |
} |
16 |
| 1784 |
|
--- |
1784 |
|
--- |
| 1785 |
SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, |
0 |
1785 |
SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, |
0 |
| 1786 |
unsigned TargetFlags) { |
--- |
1786 |
unsigned TargetFlags) { |
--- |
| 1787 |
assert((TargetFlags == 0 || isTarget) && |
0 |
1787 |
assert((TargetFlags == 0 || isTarget) && |
0 |
| 1788 |
"Cannot set target flags on target-independent jump tables"); |
--- |
1788 |
"Cannot set target flags on target-independent jump tables"); |
--- |
| 1789 |
unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; |
0 |
1789 |
unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; |
0 |
| 1790 |
FoldingSetNodeID ID; |
0 |
1790 |
FoldingSetNodeID ID; |
0 |
| 1791 |
AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt); |
0 |
1791 |
AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt); |
0 |
| 1792 |
ID.AddInteger(JTI); |
0 |
1792 |
ID.AddInteger(JTI); |
0 |
| 1793 |
ID.AddInteger(TargetFlags); |
0 |
1793 |
ID.AddInteger(TargetFlags); |
0 |
| 1794 |
void *IP = nullptr; |
0 |
1794 |
void *IP = nullptr; |
0 |
| 1795 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
1795 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
| 1796 |
return SDValue(E, 0); |
0 |
1796 |
return SDValue(E, 0); |
0 |
| 1797 |
|
--- |
1797 |
|
--- |
| 1798 |
auto *N = newSDNode(JTI, VT, isTarget, TargetFlags); |
0 |
1798 |
auto *N = newSDNode(JTI, VT, isTarget, TargetFlags); |
0 |
| 1799 |
CSEMap.InsertNode(N, IP); |
0 |
1799 |
CSEMap.InsertNode(N, IP); |
0 |
| 1800 |
InsertNode(N); |
0 |
1800 |
InsertNode(N); |
0 |
| 1801 |
return SDValue(N, 0); |
0 |
1801 |
return SDValue(N, 0); |
0 |
| 1802 |
} |
0 |
1802 |
} |
0 |
| 1803 |
|
--- |
1803 |
|
--- |
| 1804 |
SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, |
0 |
1804 |
SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, |
0 |
| 1805 |
MaybeAlign Alignment, int Offset, |
--- |
1805 |
MaybeAlign Alignment, int Offset, |
--- |
| 1806 |
bool isTarget, unsigned TargetFlags) { |
--- |
1806 |
bool isTarget, unsigned TargetFlags) { |
--- |
| 1807 |
assert((TargetFlags == 0 || isTarget) && |
0 |
1807 |
assert((TargetFlags == 0 || isTarget) && |
0 |
| 1808 |
"Cannot set target flags on target-independent globals"); |
--- |
1808 |
"Cannot set target flags on target-independent globals"); |
--- |
| 1809 |
if (!Alignment) |
0 |
1809 |
if (!Alignment) |
0 |
| 1810 |
Alignment = shouldOptForSize() |
0 |
1810 |
Alignment = shouldOptForSize() |
0 |
| 1811 |
? getDataLayout().getABITypeAlign(C->getType()) |
0 |
1811 |
? getDataLayout().getABITypeAlign(C->getType()) |
0 |
| 1812 |
: getDataLayout().getPrefTypeAlign(C->getType()); |
0 |
1812 |
: getDataLayout().getPrefTypeAlign(C->getType()); |
0 |
| 1813 |
unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; |
0 |
1813 |
unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; |
0 |
| 1814 |
FoldingSetNodeID ID; |
0 |
1814 |
FoldingSetNodeID ID; |
0 |
| 1815 |
AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt); |
0 |
1815 |
AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt); |
0 |
| 1816 |
ID.AddInteger(Alignment->value()); |
0 |
1816 |
ID.AddInteger(Alignment->value()); |
0 |
| 1817 |
ID.AddInteger(Offset); |
0 |
1817 |
ID.AddInteger(Offset); |
0 |
| 1818 |
ID.AddPointer(C); |
0 |
1818 |
ID.AddPointer(C); |
0 |
| 1819 |
ID.AddInteger(TargetFlags); |
0 |
1819 |
ID.AddInteger(TargetFlags); |
0 |
| 1820 |
void *IP = nullptr; |
0 |
1820 |
void *IP = nullptr; |
0 |
| 1821 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
1821 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
| 1822 |
return SDValue(E, 0); |
0 |
1822 |
return SDValue(E, 0); |
0 |
| 1823 |
|
--- |
1823 |
|
--- |
| 1824 |
auto *N = newSDNode(isTarget, C, VT, Offset, *Alignment, |
0 |
1824 |
auto *N = newSDNode(isTarget, C, VT, Offset, *Alignment, |
0 |
| 1825 |
TargetFlags); |
--- |
1825 |
TargetFlags); |
--- |
| 1826 |
CSEMap.InsertNode(N, IP); |
0 |
1826 |
CSEMap.InsertNode(N, IP); |
0 |
| 1827 |
InsertNode(N); |
0 |
1827 |
InsertNode(N); |
0 |
| 1828 |
SDValue V = SDValue(N, 0); |
0 |
1828 |
SDValue V = SDValue(N, 0); |
0 |
| 1829 |
NewSDValueDbgMsg(V, "Creating new constant pool: ", this); |
0 |
1829 |
NewSDValueDbgMsg(V, "Creating new constant pool: ", this); |
0 |
| 1830 |
return V; |
0 |
1830 |
return V; |
0 |
| 1831 |
} |
0 |
1831 |
} |
0 |
| 1832 |
|
--- |
1832 |
|
--- |
| 1833 |
SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, |
0 |
1833 |
SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, |
0 |
| 1834 |
MaybeAlign Alignment, int Offset, |
--- |
1834 |
MaybeAlign Alignment, int Offset, |
--- |
| 1835 |
bool isTarget, unsigned TargetFlags) { |
--- |
1835 |
bool isTarget, unsigned TargetFlags) { |
--- |
| 1836 |
assert((TargetFlags == 0 || isTarget) && |
0 |
1836 |
assert((TargetFlags == 0 || isTarget) && |
0 |
| 1837 |
"Cannot set target flags on target-independent globals"); |
--- |
1837 |
"Cannot set target flags on target-independent globals"); |
--- |
| 1838 |
if (!Alignment) |
0 |
1838 |
if (!Alignment) |
0 |
| 1839 |
Alignment = getDataLayout().getPrefTypeAlign(C->getType()); |
0 |
1839 |
Alignment = getDataLayout().getPrefTypeAlign(C->getType()); |
0 |
| 1840 |
unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; |
0 |
1840 |
unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; |
0 |
| 1841 |
FoldingSetNodeID ID; |
0 |
1841 |
FoldingSetNodeID ID; |
0 |
| 1842 |
AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt); |
0 |
1842 |
AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt); |
0 |
| 1843 |
ID.AddInteger(Alignment->value()); |
0 |
1843 |
ID.AddInteger(Alignment->value()); |
0 |
| 1844 |
ID.AddInteger(Offset); |
0 |
1844 |
ID.AddInteger(Offset); |
0 |
| 1845 |
C->addSelectionDAGCSEId(ID); |
0 |
1845 |
C->addSelectionDAGCSEId(ID); |
0 |
| 1846 |
ID.AddInteger(TargetFlags); |
0 |
1846 |
ID.AddInteger(TargetFlags); |
0 |
| 1847 |
void *IP = nullptr; |
0 |
1847 |
void *IP = nullptr; |
0 |
| 1848 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
1848 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
| 1849 |
return SDValue(E, 0); |
0 |
1849 |
return SDValue(E, 0); |
0 |
| 1850 |
|
--- |
1850 |
|
--- |
| 1851 |
auto *N = newSDNode(isTarget, C, VT, Offset, *Alignment, |
0 |
1851 |
auto *N = newSDNode(isTarget, C, VT, Offset, *Alignment, |
0 |
| 1852 |
TargetFlags); |
--- |
1852 |
TargetFlags); |
--- |
| 1853 |
CSEMap.InsertNode(N, IP); |
0 |
1853 |
CSEMap.InsertNode(N, IP); |
0 |
| 1854 |
InsertNode(N); |
0 |
1854 |
InsertNode(N); |
0 |
| 1855 |
return SDValue(N, 0); |
0 |
1855 |
return SDValue(N, 0); |
0 |
| 1856 |
} |
0 |
1856 |
} |
0 |
| 1857 |
|
--- |
1857 |
|
--- |
| 1858 |
SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset, |
0 |
1858 |
SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset, |
0 |
| 1859 |
unsigned TargetFlags) { |
--- |
1859 |
unsigned TargetFlags) { |
--- |
| 1860 |
FoldingSetNodeID ID; |
0 |
1860 |
FoldingSetNodeID ID; |
0 |
| 1861 |
AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), std::nullopt); |
0 |
1861 |
AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), std::nullopt); |
0 |
| 1862 |
ID.AddInteger(Index); |
0 |
1862 |
ID.AddInteger(Index); |
0 |
| 1863 |
ID.AddInteger(Offset); |
0 |
1863 |
ID.AddInteger(Offset); |
0 |
| 1864 |
ID.AddInteger(TargetFlags); |
0 |
1864 |
ID.AddInteger(TargetFlags); |
0 |
| 1865 |
void *IP = nullptr; |
0 |
1865 |
void *IP = nullptr; |
0 |
| 1866 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
1866 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
| 1867 |
return SDValue(E, 0); |
0 |
1867 |
return SDValue(E, 0); |
0 |
| 1868 |
|
--- |
1868 |
|
--- |
| 1869 |
auto *N = newSDNode(Index, VT, Offset, TargetFlags); |
0 |
1869 |
auto *N = newSDNode(Index, VT, Offset, TargetFlags); |
0 |
| 1870 |
CSEMap.InsertNode(N, IP); |
0 |
1870 |
CSEMap.InsertNode(N, IP); |
0 |
| 1871 |
InsertNode(N); |
0 |
1871 |
InsertNode(N); |
0 |
| 1872 |
return SDValue(N, 0); |
0 |
1872 |
return SDValue(N, 0); |
0 |
| 1873 |
} |
0 |
1873 |
} |
0 |
| 1874 |
|
--- |
1874 |
|
--- |
| 1875 |
SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { |
4 |
1875 |
SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { |
4 |
| 1876 |
FoldingSetNodeID ID; |
4 |
1876 |
FoldingSetNodeID ID; |
4 |
| 1877 |
AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), std::nullopt); |
4 |
1877 |
AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), std::nullopt); |
4 |
| 1878 |
ID.AddPointer(MBB); |
4 |
1878 |
ID.AddPointer(MBB); |
4 |
| 1879 |
void *IP = nullptr; |
4 |
1879 |
void *IP = nullptr; |
4 |
| 1880 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
4 |
1880 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
4 |
| 1881 |
return SDValue(E, 0); |
0 |
1881 |
return SDValue(E, 0); |
0 |
| 1882 |
|
--- |
1882 |
|
--- |
| 1883 |
auto *N = newSDNode(MBB); |
4 |
1883 |
auto *N = newSDNode(MBB); |
4 |
| 1884 |
CSEMap.InsertNode(N, IP); |
4 |
1884 |
CSEMap.InsertNode(N, IP); |
4 |
| 1885 |
InsertNode(N); |
4 |
1885 |
InsertNode(N); |
4 |
| 1886 |
return SDValue(N, 0); |
4 |
1886 |
return SDValue(N, 0); |
4 |
| 1887 |
} |
4 |
1887 |
} |
4 |
| 1888 |
|
--- |
1888 |
|
--- |
| 1889 |
SDValue SelectionDAG::getValueType(EVT VT) { |
0 |
1889 |
SDValue SelectionDAG::getValueType(EVT VT) { |
0 |
| 1890 |
if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >= |
0 |
1890 |
if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >= |
0 |
| 1891 |
ValueTypeNodes.size()) |
0 |
1891 |
ValueTypeNodes.size()) |
0 |
| 1892 |
ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1); |
0 |
1892 |
ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1); |
0 |
| 1893 |
|
--- |
1893 |
|
--- |
| 1894 |
SDNode *&N = VT.isExtended() ? |
0 |
1894 |
SDNode *&N = VT.isExtended() ? |
0 |
| 1895 |
ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy]; |
0 |
1895 |
ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy]; |
0 |
| 1896 |
|
--- |
1896 |
|
--- |
| 1897 |
if (N) return SDValue(N, 0); |
0 |
1897 |
if (N) return SDValue(N, 0); |
0 |
| 1898 |
N = newSDNode(VT); |
0 |
1898 |
N = newSDNode(VT); |
0 |
| 1899 |
InsertNode(N); |
0 |
1899 |
InsertNode(N); |
0 |
| 1900 |
return SDValue(N, 0); |
0 |
1900 |
return SDValue(N, 0); |
0 |
| 1901 |
} |
--- |
1901 |
} |
--- |
| 1902 |
|
--- |
1902 |
|
--- |
| 1903 |
SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { |
0 |
1903 |
SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { |
0 |
| 1904 |
SDNode *&N = ExternalSymbols[Sym]; |
0 |
1904 |
SDNode *&N = ExternalSymbols[Sym]; |
0 |
| 1905 |
if (N) return SDValue(N, 0); |
0 |
1905 |
if (N) return SDValue(N, 0); |
0 |
| 1906 |
N = newSDNode(false, Sym, 0, VT); |
0 |
1906 |
N = newSDNode(false, Sym, 0, VT); |
0 |
| 1907 |
InsertNode(N); |
0 |
1907 |
InsertNode(N); |
0 |
| 1908 |
return SDValue(N, 0); |
0 |
1908 |
return SDValue(N, 0); |
0 |
| 1909 |
} |
--- |
1909 |
} |
--- |
| 1910 |
|
--- |
1910 |
|
--- |
| 1911 |
SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) { |
0 |
1911 |
SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) { |
0 |
| 1912 |
SDNode *&N = MCSymbols[Sym]; |
0 |
1912 |
SDNode *&N = MCSymbols[Sym]; |
0 |
| 1913 |
if (N) |
0 |
1913 |
if (N) |
0 |
| 1914 |
return SDValue(N, 0); |
0 |
1914 |
return SDValue(N, 0); |
0 |
| 1915 |
N = newSDNode(Sym, VT); |
0 |
1915 |
N = newSDNode(Sym, VT); |
0 |
| 1916 |
InsertNode(N); |
0 |
1916 |
InsertNode(N); |
0 |
| 1917 |
return SDValue(N, 0); |
0 |
1917 |
return SDValue(N, 0); |
0 |
| 1918 |
} |
--- |
1918 |
} |
--- |
| 1919 |
|
--- |
1919 |
|
--- |
| 1920 |
SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, |
0 |
1920 |
SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, |
0 |
| 1921 |
unsigned TargetFlags) { |
--- |
1921 |
unsigned TargetFlags) { |
--- |
| 1922 |
SDNode *&N = |
--- |
1922 |
SDNode *&N = |
--- |
| 1923 |
TargetExternalSymbols[std::pair(Sym, TargetFlags)]; |
0 |
1923 |
TargetExternalSymbols[std::pair(Sym, TargetFlags)]; |
0 |
| 1924 |
if (N) return SDValue(N, 0); |
0 |
1924 |
if (N) return SDValue(N, 0); |
0 |
| 1925 |
N = newSDNode(true, Sym, TargetFlags, VT); |
0 |
1925 |
N = newSDNode(true, Sym, TargetFlags, VT); |
0 |
| 1926 |
InsertNode(N); |
0 |
1926 |
InsertNode(N); |
0 |
| 1927 |
return SDValue(N, 0); |
0 |
1927 |
return SDValue(N, 0); |
0 |
| 1928 |
} |
--- |
1928 |
} |
--- |
| 1929 |
|
--- |
1929 |
|
--- |
| 1930 |
SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { |
2 |
1930 |
SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { |
2 |
| 1931 |
if ((unsigned)Cond >= CondCodeNodes.size()) |
2 |
1931 |
if ((unsigned)Cond >= CondCodeNodes.size()) |
2 |
| 1932 |
CondCodeNodes.resize(Cond+1); |
2 |
1932 |
CondCodeNodes.resize(Cond+1); |
2 |
| 1933 |
|
--- |
1933 |
|
--- |
| 1934 |
if (!CondCodeNodes[Cond]) { |
2 |
1934 |
if (!CondCodeNodes[Cond]) { |
2 |
| 1935 |
auto *N = newSDNode(Cond); |
2 |
1935 |
auto *N = newSDNode(Cond); |
2 |
| 1936 |
CondCodeNodes[Cond] = N; |
2 |
1936 |
CondCodeNodes[Cond] = N; |
2 |
| 1937 |
InsertNode(N); |
2 |
1937 |
InsertNode(N); |
2 |
| 1938 |
} |
--- |
1938 |
} |
--- |
| 1939 |
|
--- |
1939 |
|
--- |
| 1940 |
return SDValue(CondCodeNodes[Cond], 0); |
2 |
1940 |
return SDValue(CondCodeNodes[Cond], 0); |
2 |
| 1941 |
} |
--- |
1941 |
} |
--- |
| 1942 |
|
--- |
1942 |
|
--- |
| 1943 |
SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm, |
0 |
1943 |
SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm, |
0 |
| 1944 |
bool ConstantFold) { |
--- |
1944 |
bool ConstantFold) { |
--- |
| 1945 |
assert(MulImm.getBitWidth() == VT.getSizeInBits() && |
0 |
1945 |
assert(MulImm.getBitWidth() == VT.getSizeInBits() && |
0 |
| 1946 |
"APInt size does not match type size!"); |
--- |
1946 |
"APInt size does not match type size!"); |
--- |
| 1947 |
|
--- |
1947 |
|
--- |
| 1948 |
if (ConstantFold) { |
0 |
1948 |
if (ConstantFold) { |
0 |
| 1949 |
const MachineFunction &MF = getMachineFunction(); |
0 |
1949 |
const MachineFunction &MF = getMachineFunction(); |
0 |
| 1950 |
const Function &F = MF.getFunction(); |
0 |
1950 |
const Function &F = MF.getFunction(); |
0 |
| 1951 |
ConstantRange CR = getVScaleRange(&F, 64); |
0 |
1951 |
ConstantRange CR = getVScaleRange(&F, 64); |
0 |
| 1952 |
if (const APInt *C = CR.getSingleElement()) |
0 |
1952 |
if (const APInt *C = CR.getSingleElement()) |
0 |
| 1953 |
return getConstant(MulImm * C->getZExtValue(), DL, VT); |
0 |
1953 |
return getConstant(MulImm * C->getZExtValue(), DL, VT); |
0 |
| 1954 |
} |
0 |
1954 |
} |
0 |
| 1955 |
|
--- |
1955 |
|
--- |
| 1956 |
return getNode(ISD::VSCALE, DL, VT, getConstant(MulImm, DL, VT)); |
0 |
1956 |
return getNode(ISD::VSCALE, DL, VT, getConstant(MulImm, DL, VT)); |
0 |
| 1957 |
} |
--- |
1957 |
} |
--- |
| 1958 |
|
--- |
1958 |
|
--- |
| 1959 |
SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, |
0 |
1959 |
SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, |
0 |
| 1960 |
bool ConstantFold) { |
--- |
1960 |
bool ConstantFold) { |
--- |
| 1961 |
if (EC.isScalable()) |
0 |
1961 |
if (EC.isScalable()) |
0 |
| 1962 |
return getVScale(DL, VT, |
0 |
1962 |
return getVScale(DL, VT, |
0 |
| 1963 |
APInt(VT.getSizeInBits(), EC.getKnownMinValue())); |
0 |
1963 |
APInt(VT.getSizeInBits(), EC.getKnownMinValue())); |
0 |
| 1964 |
|
--- |
1964 |
|
--- |
| 1965 |
return getConstant(EC.getKnownMinValue(), DL, VT); |
0 |
1965 |
return getConstant(EC.getKnownMinValue(), DL, VT); |
0 |
| 1966 |
} |
--- |
1966 |
} |
--- |
| 1967 |
|
--- |
1967 |
|
--- |
| 1968 |
SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) { |
0 |
1968 |
SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) { |
0 |
| 1969 |
APInt One(ResVT.getScalarSizeInBits(), 1); |
0 |
1969 |
APInt One(ResVT.getScalarSizeInBits(), 1); |
0 |
| 1970 |
return getStepVector(DL, ResVT, One); |
0 |
1970 |
return getStepVector(DL, ResVT, One); |
0 |
| 1971 |
} |
0 |
1971 |
} |
0 |
| 1972 |
|
--- |
1972 |
|
--- |
| 1973 |
SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal) { |
0 |
1973 |
SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal) { |
0 |
| 1974 |
assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth()); |
0 |
1974 |
assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth()); |
0 |
| 1975 |
if (ResVT.isScalableVector()) |
0 |
1975 |
if (ResVT.isScalableVector()) |
0 |
| 1976 |
return getNode( |
0 |
1976 |
return getNode( |
0 |
| 1977 |
ISD::STEP_VECTOR, DL, ResVT, |
--- |
1977 |
ISD::STEP_VECTOR, DL, ResVT, |
--- |
| 1978 |
getTargetConstant(StepVal, DL, ResVT.getVectorElementType())); |
0 |
1978 |
getTargetConstant(StepVal, DL, ResVT.getVectorElementType())); |
0 |
| 1979 |
|
--- |
1979 |
|
--- |
| 1980 |
SmallVector OpsStepConstants; |
0 |
1980 |
SmallVector OpsStepConstants; |
0 |
| 1981 |
for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++) |
0 |
1981 |
for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++) |
0 |
| 1982 |
OpsStepConstants.push_back( |
0 |
1982 |
OpsStepConstants.push_back( |
0 |
| 1983 |
getConstant(StepVal * i, DL, ResVT.getVectorElementType())); |
0 |
1983 |
getConstant(StepVal * i, DL, ResVT.getVectorElementType())); |
0 |
| 1984 |
return getBuildVector(ResVT, DL, OpsStepConstants); |
0 |
1984 |
return getBuildVector(ResVT, DL, OpsStepConstants); |
0 |
| 1985 |
} |
0 |
1985 |
} |
0 |
| 1986 |
|
--- |
1986 |
|
--- |
| 1987 |
/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that |
--- |
1987 |
/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that |
--- |
| 1988 |
/// point at N1 to point at N2 and indices that point at N2 to point at N1. |
--- |
1988 |
/// point at N1 to point at N2 and indices that point at N2 to point at N1. |
--- |
| 1989 |
static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef M) { |
0 |
1989 |
static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef M) { |
0 |
| 1990 |
std::swap(N1, N2); |
0 |
1990 |
std::swap(N1, N2); |
0 |
| 1991 |
ShuffleVectorSDNode::commuteMask(M); |
0 |
1991 |
ShuffleVectorSDNode::commuteMask(M); |
0 |
| 1992 |
} |
0 |
1992 |
} |
0 |
| 1993 |
|
--- |
1993 |
|
--- |
| 1994 |
SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, |
0 |
1994 |
SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, |
0 |
| 1995 |
SDValue N2, ArrayRef Mask) { |
--- |
1995 |
SDValue N2, ArrayRef Mask) { |
--- |
| 1996 |
assert(VT.getVectorNumElements() == Mask.size() && |
0 |
1996 |
assert(VT.getVectorNumElements() == Mask.size() && |
0 |
| 1997 |
"Must have the same number of vector elements as mask elements!"); |
--- |
1997 |
"Must have the same number of vector elements as mask elements!"); |
--- |
| 1998 |
assert(VT == N1.getValueType() && VT == N2.getValueType() && |
0 |
1998 |
assert(VT == N1.getValueType() && VT == N2.getValueType() && |
0 |
| 1999 |
"Invalid VECTOR_SHUFFLE"); |
--- |
1999 |
"Invalid VECTOR_SHUFFLE"); |
--- |
| 2000 |
|
--- |
2000 |
|
--- |
| 2001 |
// Canonicalize shuffle undef, undef -> undef |
--- |
2001 |
// Canonicalize shuffle undef, undef -> undef |
--- |
| 2002 |
if (N1.isUndef() && N2.isUndef()) |
0 |
2002 |
if (N1.isUndef() && N2.isUndef()) |
0 |
| 2003 |
return getUNDEF(VT); |
0 |
2003 |
return getUNDEF(VT); |
0 |
| 2004 |
|
--- |
2004 |
|
--- |
| 2005 |
// Validate that all indices in Mask are within the range of the elements |
--- |
2005 |
// Validate that all indices in Mask are within the range of the elements |
--- |
| 2006 |
// input to the shuffle. |
--- |
2006 |
// input to the shuffle. |
--- |
| 2007 |
int NElts = Mask.size(); |
0 |
2007 |
int NElts = Mask.size(); |
0 |
| 2008 |
assert(llvm::all_of(Mask, |
0 |
2008 |
assert(llvm::all_of(Mask, |
0 |
| 2009 |
[&](int M) { return M < (NElts * 2) && M >= -1; }) && |
--- |
2009 |
[&](int M) { return M < (NElts * 2) && M >= -1; }) && |
--- |
| 2010 |
"Index out of range"); |
--- |
2010 |
"Index out of range"); |
--- |
| 2011 |
|
--- |
2011 |
|
--- |
| 2012 |
// Copy the mask so we can do any needed cleanup. |
--- |
2012 |
// Copy the mask so we can do any needed cleanup. |
--- |
| 2013 |
SmallVector MaskVec(Mask); |
0 |
2013 |
SmallVector MaskVec(Mask); |
0 |
| 2014 |
|
--- |
2014 |
|
--- |
| 2015 |
// Canonicalize shuffle v, v -> v, undef |
--- |
2015 |
// Canonicalize shuffle v, v -> v, undef |
--- |
| 2016 |
if (N1 == N2) { |
0 |
2016 |
if (N1 == N2) { |
0 |
| 2017 |
N2 = getUNDEF(VT); |
0 |
2017 |
N2 = getUNDEF(VT); |
0 |
| 2018 |
for (int i = 0; i != NElts; ++i) |
0 |
2018 |
for (int i = 0; i != NElts; ++i) |
0 |
| 2019 |
if (MaskVec[i] >= NElts) MaskVec[i] -= NElts; |
0 |
2019 |
if (MaskVec[i] >= NElts) MaskVec[i] -= NElts; |
0 |
| 2020 |
} |
--- |
2020 |
} |
--- |
| 2021 |
|
--- |
2021 |
|
--- |
| 2022 |
// Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. |
--- |
2022 |
// Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. |
--- |
| 2023 |
if (N1.isUndef()) |
0 |
2023 |
if (N1.isUndef()) |
0 |
| 2024 |
commuteShuffle(N1, N2, MaskVec); |
0 |
2024 |
commuteShuffle(N1, N2, MaskVec); |
0 |
| 2025 |
|
--- |
2025 |
|
--- |
| 2026 |
if (TLI->hasVectorBlend()) { |
0 |
2026 |
if (TLI->hasVectorBlend()) { |
0 |
| 2027 |
// If shuffling a splat, try to blend the splat instead. We do this here so |
--- |
2027 |
// If shuffling a splat, try to blend the splat instead. We do this here so |
--- |
| 2028 |
// that even when this arises during lowering we don't have to re-handle it. |
--- |
2028 |
// that even when this arises during lowering we don't have to re-handle it. |
--- |
| 2029 |
auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) { |
0 |
2029 |
auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) { |
0 |
| 2030 |
BitVector UndefElements; |
0 |
2030 |
BitVector UndefElements; |
0 |
| 2031 |
SDValue Splat = BV->getSplatValue(&UndefElements); |
0 |
2031 |
SDValue Splat = BV->getSplatValue(&UndefElements); |
0 |
| 2032 |
if (!Splat) |
0 |
2032 |
if (!Splat) |
0 |
| 2033 |
return; |
0 |
2033 |
return; |
0 |
| 2034 |
|
--- |
2034 |
|
--- |
| 2035 |
for (int i = 0; i < NElts; ++i) { |
0 |
2035 |
for (int i = 0; i < NElts; ++i) { |
0 |
| 2036 |
if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts)) |
0 |
2036 |
if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts)) |
0 |
| 2037 |
continue; |
0 |
2037 |
continue; |
0 |
| 2038 |
|
--- |
2038 |
|
--- |
| 2039 |
// If this input comes from undef, mark it as such. |
--- |
2039 |
// If this input comes from undef, mark it as such. |
--- |
| 2040 |
if (UndefElements[MaskVec[i] - Offset]) { |
0 |
2040 |
if (UndefElements[MaskVec[i] - Offset]) { |
0 |
| 2041 |
MaskVec[i] = -1; |
0 |
2041 |
MaskVec[i] = -1; |
0 |
| 2042 |
continue; |
0 |
2042 |
continue; |
0 |
| 2043 |
} |
--- |
2043 |
} |
--- |
| 2044 |
|
--- |
2044 |
|
--- |
| 2045 |
// If we can blend a non-undef lane, use that instead. |
--- |
2045 |
// If we can blend a non-undef lane, use that instead. |
--- |
| 2046 |
if (!UndefElements[i]) |
0 |
2046 |
if (!UndefElements[i]) |
0 |
| 2047 |
MaskVec[i] = i + Offset; |
0 |
2047 |
MaskVec[i] = i + Offset; |
0 |
| 2048 |
} |
--- |
2048 |
} |
--- |
| 2049 |
}; |
0 |
2049 |
}; |
0 |
| 2050 |
if (auto *N1BV = dyn_cast(N1)) |
0 |
2050 |
if (auto *N1BV = dyn_cast(N1)) |
0 |
| 2051 |
BlendSplat(N1BV, 0); |
0 |
2051 |
BlendSplat(N1BV, 0); |
0 |
| 2052 |
if (auto *N2BV = dyn_cast(N2)) |
0 |
2052 |
if (auto *N2BV = dyn_cast(N2)) |
0 |
| 2053 |
BlendSplat(N2BV, NElts); |
0 |
2053 |
BlendSplat(N2BV, NElts); |
0 |
| 2054 |
} |
--- |
2054 |
} |
--- |
| 2055 |
|
--- |
2055 |
|
--- |
| 2056 |
// Canonicalize all index into lhs, -> shuffle lhs, undef |
--- |
2056 |
// Canonicalize all index into lhs, -> shuffle lhs, undef |
--- |
| 2057 |
// Canonicalize all index into rhs, -> shuffle rhs, undef |
--- |
2057 |
// Canonicalize all index into rhs, -> shuffle rhs, undef |
--- |
| 2058 |
bool AllLHS = true, AllRHS = true; |
0 |
2058 |
bool AllLHS = true, AllRHS = true; |
0 |
| 2059 |
bool N2Undef = N2.isUndef(); |
0 |
2059 |
bool N2Undef = N2.isUndef(); |
0 |
| 2060 |
for (int i = 0; i != NElts; ++i) { |
0 |
2060 |
for (int i = 0; i != NElts; ++i) { |
0 |
| 2061 |
if (MaskVec[i] >= NElts) { |
0 |
2061 |
if (MaskVec[i] >= NElts) { |
0 |
| 2062 |
if (N2Undef) |
0 |
2062 |
if (N2Undef) |
0 |
| 2063 |
MaskVec[i] = -1; |
0 |
2063 |
MaskVec[i] = -1; |
0 |
| 2064 |
else |
--- |
2064 |
else |
--- |
| 2065 |
AllLHS = false; |
0 |
2065 |
AllLHS = false; |
0 |
| 2066 |
} else if (MaskVec[i] >= 0) { |
0 |
2066 |
} else if (MaskVec[i] >= 0) { |
0 |
| 2067 |
AllRHS = false; |
0 |
2067 |
AllRHS = false; |
0 |
| 2068 |
} |
--- |
2068 |
} |
--- |
| 2069 |
} |
--- |
2069 |
} |
--- |
| 2070 |
if (AllLHS && AllRHS) |
0 |
2070 |
if (AllLHS && AllRHS) |
0 |
| 2071 |
return getUNDEF(VT); |
0 |
2071 |
return getUNDEF(VT); |
0 |
| 2072 |
if (AllLHS && !N2Undef) |
0 |
2072 |
if (AllLHS && !N2Undef) |
0 |
| 2073 |
N2 = getUNDEF(VT); |
0 |
2073 |
N2 = getUNDEF(VT); |
0 |
| 2074 |
if (AllRHS) { |
0 |
2074 |
if (AllRHS) { |
0 |
| 2075 |
N1 = getUNDEF(VT); |
0 |
2075 |
N1 = getUNDEF(VT); |
0 |
| 2076 |
commuteShuffle(N1, N2, MaskVec); |
0 |
2076 |
commuteShuffle(N1, N2, MaskVec); |
0 |
| 2077 |
} |
--- |
2077 |
} |
--- |
| 2078 |
// Reset our undef status after accounting for the mask. |
--- |
2078 |
// Reset our undef status after accounting for the mask. |
--- |
| 2079 |
N2Undef = N2.isUndef(); |
0 |
2079 |
N2Undef = N2.isUndef(); |
0 |
| 2080 |
// Re-check whether both sides ended up undef. |
--- |
2080 |
// Re-check whether both sides ended up undef. |
--- |
| 2081 |
if (N1.isUndef() && N2Undef) |
0 |
2081 |
if (N1.isUndef() && N2Undef) |
0 |
| 2082 |
return getUNDEF(VT); |
0 |
2082 |
return getUNDEF(VT); |
0 |
| 2083 |
|
--- |
2083 |
|
--- |
| 2084 |
// If Identity shuffle return that node. |
--- |
2084 |
// If Identity shuffle return that node. |
--- |
| 2085 |
bool Identity = true, AllSame = true; |
0 |
2085 |
bool Identity = true, AllSame = true; |
0 |
| 2086 |
for (int i = 0; i != NElts; ++i) { |
0 |
2086 |
for (int i = 0; i != NElts; ++i) { |
0 |
| 2087 |
if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false; |
0 |
2087 |
if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false; |
0 |
| 2088 |
if (MaskVec[i] != MaskVec[0]) AllSame = false; |
0 |
2088 |
if (MaskVec[i] != MaskVec[0]) AllSame = false; |
0 |
| 2089 |
} |
--- |
2089 |
} |
--- |
| 2090 |
if (Identity && NElts) |
0 |
2090 |
if (Identity && NElts) |
0 |
| 2091 |
return N1; |
0 |
2091 |
return N1; |
0 |
| 2092 |
|
--- |
2092 |
|
--- |
| 2093 |
// Shuffling a constant splat doesn't change the result. |
--- |
2093 |
// Shuffling a constant splat doesn't change the result. |
--- |
| 2094 |
if (N2Undef) { |
0 |
2094 |
if (N2Undef) { |
0 |
| 2095 |
SDValue V = N1; |
0 |
2095 |
SDValue V = N1; |
0 |
| 2096 |
|
--- |
2096 |
|
--- |
| 2097 |
// Look through any bitcasts. We check that these don't change the number |
--- |
2097 |
// Look through any bitcasts. We check that these don't change the number |
--- |
| 2098 |
// (and size) of elements and just changes their types. |
--- |
2098 |
// (and size) of elements and just changes their types. |
--- |
| 2099 |
while (V.getOpcode() == ISD::BITCAST) |
0 |
2099 |
while (V.getOpcode() == ISD::BITCAST) |
0 |
| 2100 |
V = V->getOperand(0); |
0 |
2100 |
V = V->getOperand(0); |
0 |
| 2101 |
|
--- |
2101 |
|
--- |
| 2102 |
// A splat should always show up as a build vector node. |
--- |
2102 |
// A splat should always show up as a build vector node. |
--- |
| 2103 |
if (auto *BV = dyn_cast(V)) { |
0 |
2103 |
if (auto *BV = dyn_cast(V)) { |
0 |
| 2104 |
BitVector UndefElements; |
0 |
2104 |
BitVector UndefElements; |
0 |
| 2105 |
SDValue Splat = BV->getSplatValue(&UndefElements); |
0 |
2105 |
SDValue Splat = BV->getSplatValue(&UndefElements); |
0 |
| 2106 |
// If this is a splat of an undef, shuffling it is also undef. |
--- |
2106 |
// If this is a splat of an undef, shuffling it is also undef. |
--- |
| 2107 |
if (Splat && Splat.isUndef()) |
0 |
2107 |
if (Splat && Splat.isUndef()) |
0 |
| 2108 |
return getUNDEF(VT); |
0 |
2108 |
return getUNDEF(VT); |
0 |
| 2109 |
|
--- |
2109 |
|
--- |
| 2110 |
bool SameNumElts = |
--- |
2110 |
bool SameNumElts = |
--- |
| 2111 |
V.getValueType().getVectorNumElements() == VT.getVectorNumElements(); |
0 |
2111 |
V.getValueType().getVectorNumElements() == VT.getVectorNumElements(); |
0 |
| 2112 |
|
--- |
2112 |
|
--- |
| 2113 |
// We only have a splat which can skip shuffles if there is a splatted |
--- |
2113 |
// We only have a splat which can skip shuffles if there is a splatted |
--- |
| 2114 |
// value and no undef lanes rearranged by the shuffle. |
--- |
2114 |
// value and no undef lanes rearranged by the shuffle. |
--- |
| 2115 |
if (Splat && UndefElements.none()) { |
0 |
2115 |
if (Splat && UndefElements.none()) { |
0 |
| 2116 |
// Splat of , return , provided that the |
--- |
2116 |
// Splat of , return , provided that the |
--- |
| 2117 |
// number of elements match or the value splatted is a zero constant. |
--- |
2117 |
// number of elements match or the value splatted is a zero constant. |
--- |
| 2118 |
if (SameNumElts) |
0 |
2118 |
if (SameNumElts) |
0 |
| 2119 |
return N1; |
0 |
2119 |
return N1; |
0 |
| 2120 |
if (auto *C = dyn_cast(Splat)) |
0 |
2120 |
if (auto *C = dyn_cast(Splat)) |
0 |
| 2121 |
if (C->isZero()) |
0 |
2121 |
if (C->isZero()) |
0 |
| 2122 |
return N1; |
0 |
2122 |
return N1; |
0 |
| 2123 |
} |
--- |
2123 |
} |
--- |
| 2124 |
|
--- |
2124 |
|
--- |
| 2125 |
// If the shuffle itself creates a splat, build the vector directly. |
--- |
2125 |
// If the shuffle itself creates a splat, build the vector directly. |
--- |
| 2126 |
if (AllSame && SameNumElts) { |
0 |
2126 |
if (AllSame && SameNumElts) { |
0 |
| 2127 |
EVT BuildVT = BV->getValueType(0); |
0 |
2127 |
EVT BuildVT = BV->getValueType(0); |
0 |
| 2128 |
const SDValue &Splatted = BV->getOperand(MaskVec[0]); |
0 |
2128 |
const SDValue &Splatted = BV->getOperand(MaskVec[0]); |
0 |
| 2129 |
SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted); |
0 |
2129 |
SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted); |
0 |
| 2130 |
|
--- |
2130 |
|
--- |
| 2131 |
// We may have jumped through bitcasts, so the type of the |
--- |
2131 |
// We may have jumped through bitcasts, so the type of the |
--- |
| 2132 |
// BUILD_VECTOR may not match the type of the shuffle. |
--- |
2132 |
// BUILD_VECTOR may not match the type of the shuffle. |
--- |
| 2133 |
if (BuildVT != VT) |
0 |
2133 |
if (BuildVT != VT) |
0 |
| 2134 |
NewBV = getNode(ISD::BITCAST, dl, VT, NewBV); |
0 |
2134 |
NewBV = getNode(ISD::BITCAST, dl, VT, NewBV); |
0 |
| 2135 |
return NewBV; |
0 |
2135 |
return NewBV; |
0 |
| 2136 |
} |
--- |
2136 |
} |
--- |
| 2137 |
} |
0 |
2137 |
} |
0 |
| 2138 |
} |
--- |
2138 |
} |
--- |
| 2139 |
|
--- |
2139 |
|
--- |
| 2140 |
FoldingSetNodeID ID; |
0 |
2140 |
FoldingSetNodeID ID; |
0 |
| 2141 |
SDValue Ops[2] = { N1, N2 }; |
0 |
2141 |
SDValue Ops[2] = { N1, N2 }; |
0 |
| 2142 |
AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops); |
0 |
2142 |
AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops); |
0 |
| 2143 |
for (int i = 0; i != NElts; ++i) |
0 |
2143 |
for (int i = 0; i != NElts; ++i) |
0 |
| 2144 |
ID.AddInteger(MaskVec[i]); |
0 |
2144 |
ID.AddInteger(MaskVec[i]); |
0 |
| 2145 |
|
--- |
2145 |
|
--- |
| 2146 |
void* IP = nullptr; |
0 |
2146 |
void* IP = nullptr; |
0 |
| 2147 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) |
0 |
2147 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) |
0 |
| 2148 |
return SDValue(E, 0); |
0 |
2148 |
return SDValue(E, 0); |
0 |
| 2149 |
|
--- |
2149 |
|
--- |
| 2150 |
// Allocate the mask array for the node out of the BumpPtrAllocator, since |
--- |
2150 |
// Allocate the mask array for the node out of the BumpPtrAllocator, since |
--- |
| 2151 |
// SDNode doesn't have access to it. This memory will be "leaked" when |
--- |
2151 |
// SDNode doesn't have access to it. This memory will be "leaked" when |
--- |
| 2152 |
// the node is deallocated, but recovered when the NodeAllocator is released. |
--- |
2152 |
// the node is deallocated, but recovered when the NodeAllocator is released. |
--- |
| 2153 |
int *MaskAlloc = OperandAllocator.Allocate(NElts); |
0 |
2153 |
int *MaskAlloc = OperandAllocator.Allocate(NElts); |
0 |
| 2154 |
llvm::copy(MaskVec, MaskAlloc); |
0 |
2154 |
llvm::copy(MaskVec, MaskAlloc); |
0 |
| 2155 |
|
--- |
2155 |
|
--- |
| 2156 |
auto *N = newSDNode(VT, dl.getIROrder(), |
0 |
2156 |
auto *N = newSDNode(VT, dl.getIROrder(), |
0 |
| 2157 |
dl.getDebugLoc(), MaskAlloc); |
--- |
2157 |
dl.getDebugLoc(), MaskAlloc); |
--- |
| 2158 |
createOperands(N, Ops); |
0 |
2158 |
createOperands(N, Ops); |
0 |
| 2159 |
|
--- |
2159 |
|
--- |
| 2160 |
CSEMap.InsertNode(N, IP); |
0 |
2160 |
CSEMap.InsertNode(N, IP); |
0 |
| 2161 |
InsertNode(N); |
0 |
2161 |
InsertNode(N); |
0 |
| 2162 |
SDValue V = SDValue(N, 0); |
0 |
2162 |
SDValue V = SDValue(N, 0); |
0 |
| 2163 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
2163 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 2164 |
return V; |
0 |
2164 |
return V; |
0 |
| 2165 |
} |
0 |
2165 |
} |
0 |
| 2166 |
|
--- |
2166 |
|
--- |
| 2167 |
SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) { |
0 |
2167 |
SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) { |
0 |
| 2168 |
EVT VT = SV.getValueType(0); |
0 |
2168 |
EVT VT = SV.getValueType(0); |
0 |
| 2169 |
SmallVector MaskVec(SV.getMask()); |
0 |
2169 |
SmallVector MaskVec(SV.getMask()); |
0 |
| 2170 |
ShuffleVectorSDNode::commuteMask(MaskVec); |
0 |
2170 |
ShuffleVectorSDNode::commuteMask(MaskVec); |
0 |
| 2171 |
|
--- |
2171 |
|
--- |
| 2172 |
SDValue Op0 = SV.getOperand(0); |
0 |
2172 |
SDValue Op0 = SV.getOperand(0); |
0 |
| 2173 |
SDValue Op1 = SV.getOperand(1); |
0 |
2173 |
SDValue Op1 = SV.getOperand(1); |
0 |
| 2174 |
return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec); |
0 |
2174 |
return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec); |
0 |
| 2175 |
} |
0 |
2175 |
} |
0 |
| 2176 |
|
--- |
2176 |
|
--- |
| 2177 |
SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { |
5 |
2177 |
SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { |
5 |
| 2178 |
FoldingSetNodeID ID; |
5 |
2178 |
FoldingSetNodeID ID; |
5 |
| 2179 |
AddNodeIDNode(ID, ISD::Register, getVTList(VT), std::nullopt); |
5 |
2179 |
AddNodeIDNode(ID, ISD::Register, getVTList(VT), std::nullopt); |
5 |
| 2180 |
ID.AddInteger(RegNo); |
5 |
2180 |
ID.AddInteger(RegNo); |
5 |
| 2181 |
void *IP = nullptr; |
5 |
2181 |
void *IP = nullptr; |
5 |
| 2182 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
5 |
2182 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
5 |
| 2183 |
return SDValue(E, 0); |
1 |
2183 |
return SDValue(E, 0); |
1 |
| 2184 |
|
--- |
2184 |
|
--- |
| 2185 |
auto *N = newSDNode(RegNo, VT); |
4 |
2185 |
auto *N = newSDNode(RegNo, VT); |
4 |
| 2186 |
N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA); |
4 |
2186 |
N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA); |
4 |
| 2187 |
CSEMap.InsertNode(N, IP); |
4 |
2187 |
CSEMap.InsertNode(N, IP); |
4 |
| 2188 |
InsertNode(N); |
4 |
2188 |
InsertNode(N); |
4 |
| 2189 |
return SDValue(N, 0); |
4 |
2189 |
return SDValue(N, 0); |
4 |
| 2190 |
} |
5 |
2190 |
} |
5 |
| 2191 |
|
--- |
2191 |
|
--- |
| 2192 |
SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) { |
0 |
2192 |
SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) { |
0 |
| 2193 |
FoldingSetNodeID ID; |
0 |
2193 |
FoldingSetNodeID ID; |
0 |
| 2194 |
AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), std::nullopt); |
0 |
2194 |
AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), std::nullopt); |
0 |
| 2195 |
ID.AddPointer(RegMask); |
0 |
2195 |
ID.AddPointer(RegMask); |
0 |
| 2196 |
void *IP = nullptr; |
0 |
2196 |
void *IP = nullptr; |
0 |
| 2197 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
2197 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
| 2198 |
return SDValue(E, 0); |
0 |
2198 |
return SDValue(E, 0); |
0 |
| 2199 |
|
--- |
2199 |
|
--- |
| 2200 |
auto *N = newSDNode(RegMask); |
0 |
2200 |
auto *N = newSDNode(RegMask); |
0 |
| 2201 |
CSEMap.InsertNode(N, IP); |
0 |
2201 |
CSEMap.InsertNode(N, IP); |
0 |
| 2202 |
InsertNode(N); |
0 |
2202 |
InsertNode(N); |
0 |
| 2203 |
return SDValue(N, 0); |
0 |
2203 |
return SDValue(N, 0); |
0 |
| 2204 |
} |
0 |
2204 |
} |
0 |
| 2205 |
|
--- |
2205 |
|
--- |
| 2206 |
SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root, |
0 |
2206 |
SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root, |
0 |
| 2207 |
MCSymbol *Label) { |
--- |
2207 |
MCSymbol *Label) { |
--- |
| 2208 |
return getLabelNode(ISD::EH_LABEL, dl, Root, Label); |
0 |
2208 |
return getLabelNode(ISD::EH_LABEL, dl, Root, Label); |
0 |
| 2209 |
} |
--- |
2209 |
} |
--- |
| 2210 |
|
--- |
2210 |
|
--- |
| 2211 |
SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl, |
0 |
2211 |
SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl, |
0 |
| 2212 |
SDValue Root, MCSymbol *Label) { |
--- |
2212 |
SDValue Root, MCSymbol *Label) { |
--- |
| 2213 |
FoldingSetNodeID ID; |
0 |
2213 |
FoldingSetNodeID ID; |
0 |
| 2214 |
SDValue Ops[] = { Root }; |
0 |
2214 |
SDValue Ops[] = { Root }; |
0 |
| 2215 |
AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops); |
0 |
2215 |
AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops); |
0 |
| 2216 |
ID.AddPointer(Label); |
0 |
2216 |
ID.AddPointer(Label); |
0 |
| 2217 |
void *IP = nullptr; |
0 |
2217 |
void *IP = nullptr; |
0 |
| 2218 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
2218 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
| 2219 |
return SDValue(E, 0); |
0 |
2219 |
return SDValue(E, 0); |
0 |
| 2220 |
|
--- |
2220 |
|
--- |
| 2221 |
auto *N = |
--- |
2221 |
auto *N = |
--- |
| 2222 |
newSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label); |
0 |
2222 |
newSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label); |
0 |
| 2223 |
createOperands(N, Ops); |
0 |
2223 |
createOperands(N, Ops); |
0 |
| 2224 |
|
--- |
2224 |
|
--- |
| 2225 |
CSEMap.InsertNode(N, IP); |
0 |
2225 |
CSEMap.InsertNode(N, IP); |
0 |
| 2226 |
InsertNode(N); |
0 |
2226 |
InsertNode(N); |
0 |
| 2227 |
return SDValue(N, 0); |
0 |
2227 |
return SDValue(N, 0); |
0 |
| 2228 |
} |
0 |
2228 |
} |
0 |
| 2229 |
|
--- |
2229 |
|
--- |
| 2230 |
SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, |
0 |
2230 |
SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, |
0 |
| 2231 |
int64_t Offset, bool isTarget, |
--- |
2231 |
int64_t Offset, bool isTarget, |
--- |
| 2232 |
unsigned TargetFlags) { |
--- |
2232 |
unsigned TargetFlags) { |
--- |
| 2233 |
unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; |
0 |
2233 |
unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; |
0 |
| 2234 |
|
--- |
2234 |
|
--- |
| 2235 |
FoldingSetNodeID ID; |
0 |
2235 |
FoldingSetNodeID ID; |
0 |
| 2236 |
AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt); |
0 |
2236 |
AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt); |
0 |
| 2237 |
ID.AddPointer(BA); |
0 |
2237 |
ID.AddPointer(BA); |
0 |
| 2238 |
ID.AddInteger(Offset); |
0 |
2238 |
ID.AddInteger(Offset); |
0 |
| 2239 |
ID.AddInteger(TargetFlags); |
0 |
2239 |
ID.AddInteger(TargetFlags); |
0 |
| 2240 |
void *IP = nullptr; |
0 |
2240 |
void *IP = nullptr; |
0 |
| 2241 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
2241 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
| 2242 |
return SDValue(E, 0); |
0 |
2242 |
return SDValue(E, 0); |
0 |
| 2243 |
|
--- |
2243 |
|
--- |
| 2244 |
auto *N = newSDNode(Opc, VT, BA, Offset, TargetFlags); |
0 |
2244 |
auto *N = newSDNode(Opc, VT, BA, Offset, TargetFlags); |
0 |
| 2245 |
CSEMap.InsertNode(N, IP); |
0 |
2245 |
CSEMap.InsertNode(N, IP); |
0 |
| 2246 |
InsertNode(N); |
0 |
2246 |
InsertNode(N); |
0 |
| 2247 |
return SDValue(N, 0); |
0 |
2247 |
return SDValue(N, 0); |
0 |
| 2248 |
} |
0 |
2248 |
} |
0 |
| 2249 |
|
--- |
2249 |
|
--- |
| 2250 |
SDValue SelectionDAG::getSrcValue(const Value *V) { |
0 |
2250 |
SDValue SelectionDAG::getSrcValue(const Value *V) { |
0 |
| 2251 |
FoldingSetNodeID ID; |
0 |
2251 |
FoldingSetNodeID ID; |
0 |
| 2252 |
AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), std::nullopt); |
0 |
2252 |
AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), std::nullopt); |
0 |
| 2253 |
ID.AddPointer(V); |
0 |
2253 |
ID.AddPointer(V); |
0 |
| 2254 |
|
--- |
2254 |
|
--- |
| 2255 |
void *IP = nullptr; |
0 |
2255 |
void *IP = nullptr; |
0 |
| 2256 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
2256 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
| 2257 |
return SDValue(E, 0); |
0 |
2257 |
return SDValue(E, 0); |
0 |
| 2258 |
|
--- |
2258 |
|
--- |
| 2259 |
auto *N = newSDNode(V); |
0 |
2259 |
auto *N = newSDNode(V); |
0 |
| 2260 |
CSEMap.InsertNode(N, IP); |
0 |
2260 |
CSEMap.InsertNode(N, IP); |
0 |
| 2261 |
InsertNode(N); |
0 |
2261 |
InsertNode(N); |
0 |
| 2262 |
return SDValue(N, 0); |
0 |
2262 |
return SDValue(N, 0); |
0 |
| 2263 |
} |
0 |
2263 |
} |
0 |
| 2264 |
|
--- |
2264 |
|
--- |
| 2265 |
SDValue SelectionDAG::getMDNode(const MDNode *MD) { |
0 |
2265 |
SDValue SelectionDAG::getMDNode(const MDNode *MD) { |
0 |
| 2266 |
FoldingSetNodeID ID; |
0 |
2266 |
FoldingSetNodeID ID; |
0 |
| 2267 |
AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), std::nullopt); |
0 |
2267 |
AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), std::nullopt); |
0 |
| 2268 |
ID.AddPointer(MD); |
0 |
2268 |
ID.AddPointer(MD); |
0 |
| 2269 |
|
--- |
2269 |
|
--- |
| 2270 |
void *IP = nullptr; |
0 |
2270 |
void *IP = nullptr; |
0 |
| 2271 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
2271 |
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
0 |
| 2272 |
return SDValue(E, 0); |
0 |
2272 |
return SDValue(E, 0); |
0 |
| 2273 |
|
--- |
2273 |
|
--- |
| 2274 |
auto *N = newSDNode(MD); |
0 |
2274 |
auto *N = newSDNode(MD); |
0 |
| 2275 |
CSEMap.InsertNode(N, IP); |
0 |
2275 |
CSEMap.InsertNode(N, IP); |
0 |
| 2276 |
InsertNode(N); |
0 |
2276 |
InsertNode(N); |
0 |
| 2277 |
return SDValue(N, 0); |
0 |
2277 |
return SDValue(N, 0); |
0 |
| 2278 |
} |
0 |
2278 |
} |
0 |
| 2279 |
|
--- |
2279 |
|
--- |
| 2280 |
SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) { |
0 |
2280 |
SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) { |
0 |
| 2281 |
if (VT == V.getValueType()) |
0 |
2281 |
if (VT == V.getValueType()) |
0 |
| 2282 |
return V; |
0 |
2282 |
return V; |
0 |
| 2283 |
|
--- |
2283 |
|
--- |
| 2284 |
return getNode(ISD::BITCAST, SDLoc(V), VT, V); |
0 |
2284 |
return getNode(ISD::BITCAST, SDLoc(V), VT, V); |
0 |
| 2285 |
} |
--- |
2285 |
} |
--- |
| 2286 |
|
--- |
2286 |
|
--- |
| 2287 |
SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, |
0 |
2287 |
SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, |
0 |
| 2288 |
unsigned SrcAS, unsigned DestAS) { |
--- |
2288 |
unsigned SrcAS, unsigned DestAS) { |
--- |
| 2289 |
SDValue Ops[] = {Ptr}; |
0 |
2289 |
SDValue Ops[] = {Ptr}; |
0 |
| 2290 |
FoldingSetNodeID ID; |
0 |
2290 |
FoldingSetNodeID ID; |
0 |
| 2291 |
AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops); |
0 |
2291 |
AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops); |
0 |
| 2292 |
ID.AddInteger(SrcAS); |
0 |
2292 |
ID.AddInteger(SrcAS); |
0 |
| 2293 |
ID.AddInteger(DestAS); |
0 |
2293 |
ID.AddInteger(DestAS); |
0 |
| 2294 |
|
--- |
2294 |
|
--- |
| 2295 |
void *IP = nullptr; |
0 |
2295 |
void *IP = nullptr; |
0 |
| 2296 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) |
0 |
2296 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) |
0 |
| 2297 |
return SDValue(E, 0); |
0 |
2297 |
return SDValue(E, 0); |
0 |
| 2298 |
|
--- |
2298 |
|
--- |
| 2299 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), |
0 |
2299 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), |
0 |
| 2300 |
VT, SrcAS, DestAS); |
--- |
2300 |
VT, SrcAS, DestAS); |
--- |
| 2301 |
createOperands(N, Ops); |
0 |
2301 |
createOperands(N, Ops); |
0 |
| 2302 |
|
--- |
2302 |
|
--- |
| 2303 |
CSEMap.InsertNode(N, IP); |
0 |
2303 |
CSEMap.InsertNode(N, IP); |
0 |
| 2304 |
InsertNode(N); |
0 |
2304 |
InsertNode(N); |
0 |
| 2305 |
return SDValue(N, 0); |
0 |
2305 |
return SDValue(N, 0); |
0 |
| 2306 |
} |
0 |
2306 |
} |
0 |
| 2307 |
|
--- |
2307 |
|
--- |
| 2308 |
SDValue SelectionDAG::getFreeze(SDValue V) { |
0 |
2308 |
SDValue SelectionDAG::getFreeze(SDValue V) { |
0 |
| 2309 |
return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V); |
0 |
2309 |
return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V); |
0 |
| 2310 |
} |
--- |
2310 |
} |
--- |
| 2311 |
|
--- |
2311 |
|
--- |
| 2312 |
/// getShiftAmountOperand - Return the specified value casted to |
--- |
2312 |
/// getShiftAmountOperand - Return the specified value casted to |
--- |
| 2313 |
/// the target's desired shift amount type. |
--- |
2313 |
/// the target's desired shift amount type. |
--- |
| 2314 |
SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { |
0 |
2314 |
SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { |
0 |
| 2315 |
EVT OpTy = Op.getValueType(); |
0 |
2315 |
EVT OpTy = Op.getValueType(); |
0 |
| 2316 |
EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout()); |
0 |
2316 |
EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout()); |
0 |
| 2317 |
if (OpTy == ShTy || OpTy.isVector()) return Op; |
0 |
2317 |
if (OpTy == ShTy || OpTy.isVector()) return Op; |
0 |
| 2318 |
|
--- |
2318 |
|
--- |
| 2319 |
return getZExtOrTrunc(Op, SDLoc(Op), ShTy); |
0 |
2319 |
return getZExtOrTrunc(Op, SDLoc(Op), ShTy); |
0 |
| 2320 |
} |
--- |
2320 |
} |
--- |
| 2321 |
|
--- |
2321 |
|
--- |
| 2322 |
SDValue SelectionDAG::expandVAArg(SDNode *Node) { |
0 |
2322 |
SDValue SelectionDAG::expandVAArg(SDNode *Node) { |
0 |
| 2323 |
SDLoc dl(Node); |
0 |
2323 |
SDLoc dl(Node); |
0 |
| 2324 |
const TargetLowering &TLI = getTargetLoweringInfo(); |
0 |
2324 |
const TargetLowering &TLI = getTargetLoweringInfo(); |
0 |
| 2325 |
const Value *V = cast(Node->getOperand(2))->getValue(); |
0 |
2325 |
const Value *V = cast(Node->getOperand(2))->getValue(); |
0 |
| 2326 |
EVT VT = Node->getValueType(0); |
0 |
2326 |
EVT VT = Node->getValueType(0); |
0 |
| 2327 |
SDValue Tmp1 = Node->getOperand(0); |
0 |
2327 |
SDValue Tmp1 = Node->getOperand(0); |
0 |
| 2328 |
SDValue Tmp2 = Node->getOperand(1); |
0 |
2328 |
SDValue Tmp2 = Node->getOperand(1); |
0 |
| 2329 |
const MaybeAlign MA(Node->getConstantOperandVal(3)); |
0 |
2329 |
const MaybeAlign MA(Node->getConstantOperandVal(3)); |
0 |
| 2330 |
|
--- |
2330 |
|
--- |
| 2331 |
SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1, |
0 |
2331 |
SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1, |
0 |
| 2332 |
Tmp2, MachinePointerInfo(V)); |
--- |
2332 |
Tmp2, MachinePointerInfo(V)); |
--- |
| 2333 |
SDValue VAList = VAListLoad; |
0 |
2333 |
SDValue VAList = VAListLoad; |
0 |
| 2334 |
|
--- |
2334 |
|
--- |
| 2335 |
if (MA && *MA > TLI.getMinStackArgumentAlignment()) { |
0 |
2335 |
if (MA && *MA > TLI.getMinStackArgumentAlignment()) { |
0 |
| 2336 |
VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, |
0 |
2336 |
VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, |
0 |
| 2337 |
getConstant(MA->value() - 1, dl, VAList.getValueType())); |
0 |
2337 |
getConstant(MA->value() - 1, dl, VAList.getValueType())); |
0 |
| 2338 |
|
--- |
2338 |
|
--- |
| 2339 |
VAList = |
0 |
2339 |
VAList = |
0 |
| 2340 |
getNode(ISD::AND, dl, VAList.getValueType(), VAList, |
0 |
2340 |
getNode(ISD::AND, dl, VAList.getValueType(), VAList, |
0 |
| 2341 |
getConstant(-(int64_t)MA->value(), dl, VAList.getValueType())); |
0 |
2341 |
getConstant(-(int64_t)MA->value(), dl, VAList.getValueType())); |
0 |
| 2342 |
} |
--- |
2342 |
} |
--- |
| 2343 |
|
--- |
2343 |
|
--- |
| 2344 |
// Increment the pointer, VAList, to the next vaarg |
--- |
2344 |
// Increment the pointer, VAList, to the next vaarg |
--- |
| 2345 |
Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, |
0 |
2345 |
Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, |
0 |
| 2346 |
getConstant(getDataLayout().getTypeAllocSize( |
0 |
2346 |
getConstant(getDataLayout().getTypeAllocSize( |
0 |
| 2347 |
VT.getTypeForEVT(*getContext())), |
0 |
2347 |
VT.getTypeForEVT(*getContext())), |
0 |
| 2348 |
dl, VAList.getValueType())); |
--- |
2348 |
dl, VAList.getValueType())); |
--- |
| 2349 |
// Store the incremented VAList to the legalized pointer |
--- |
2349 |
// Store the incremented VAList to the legalized pointer |
--- |
| 2350 |
Tmp1 = |
0 |
2350 |
Tmp1 = |
0 |
| 2351 |
getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V)); |
0 |
2351 |
getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V)); |
0 |
| 2352 |
// Load the actual argument out of the pointer VAList |
--- |
2352 |
// Load the actual argument out of the pointer VAList |
--- |
| 2353 |
return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo()); |
0 |
2353 |
return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo()); |
0 |
| 2354 |
} |
0 |
2354 |
} |
0 |
| 2355 |
|
--- |
2355 |
|
--- |
| 2356 |
SDValue SelectionDAG::expandVACopy(SDNode *Node) { |
0 |
2356 |
SDValue SelectionDAG::expandVACopy(SDNode *Node) { |
0 |
| 2357 |
SDLoc dl(Node); |
0 |
2357 |
SDLoc dl(Node); |
0 |
| 2358 |
const TargetLowering &TLI = getTargetLoweringInfo(); |
0 |
2358 |
const TargetLowering &TLI = getTargetLoweringInfo(); |
0 |
| 2359 |
// This defaults to loading a pointer from the input and storing it to the |
--- |
2359 |
// This defaults to loading a pointer from the input and storing it to the |
--- |
| 2360 |
// output, returning the chain. |
--- |
2360 |
// output, returning the chain. |
--- |
| 2361 |
const Value *VD = cast(Node->getOperand(3))->getValue(); |
0 |
2361 |
const Value *VD = cast(Node->getOperand(3))->getValue(); |
0 |
| 2362 |
const Value *VS = cast(Node->getOperand(4))->getValue(); |
0 |
2362 |
const Value *VS = cast(Node->getOperand(4))->getValue(); |
0 |
| 2363 |
SDValue Tmp1 = |
--- |
2363 |
SDValue Tmp1 = |
--- |
| 2364 |
getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0), |
0 |
2364 |
getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0), |
0 |
| 2365 |
Node->getOperand(2), MachinePointerInfo(VS)); |
0 |
2365 |
Node->getOperand(2), MachinePointerInfo(VS)); |
0 |
| 2366 |
return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1), |
0 |
2366 |
return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1), |
0 |
| 2367 |
MachinePointerInfo(VD)); |
0 |
2367 |
MachinePointerInfo(VD)); |
0 |
| 2368 |
} |
0 |
2368 |
} |
0 |
| 2369 |
|
--- |
2369 |
|
--- |
| 2370 |
Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) { |
0 |
2370 |
Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) { |
0 |
| 2371 |
const DataLayout &DL = getDataLayout(); |
0 |
2371 |
const DataLayout &DL = getDataLayout(); |
0 |
| 2372 |
Type *Ty = VT.getTypeForEVT(*getContext()); |
0 |
2372 |
Type *Ty = VT.getTypeForEVT(*getContext()); |
0 |
| 2373 |
Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty); |
0 |
2373 |
Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty); |
0 |
| 2374 |
|
--- |
2374 |
|
--- |
| 2375 |
if (TLI->isTypeLegal(VT) || !VT.isVector()) |
0 |
2375 |
if (TLI->isTypeLegal(VT) || !VT.isVector()) |
0 |
| 2376 |
return RedAlign; |
0 |
2376 |
return RedAlign; |
0 |
| 2377 |
|
--- |
2377 |
|
--- |
| 2378 |
const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); |
0 |
2378 |
const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); |
0 |
| 2379 |
const Align StackAlign = TFI->getStackAlign(); |
0 |
2379 |
const Align StackAlign = TFI->getStackAlign(); |
0 |
| 2380 |
|
--- |
2380 |
|
--- |
| 2381 |
// See if we can choose a smaller ABI alignment in cases where it's an |
--- |
2381 |
// See if we can choose a smaller ABI alignment in cases where it's an |
--- |
| 2382 |
// illegal vector type that will get broken down. |
--- |
2382 |
// illegal vector type that will get broken down. |
--- |
| 2383 |
if (RedAlign > StackAlign) { |
0 |
2383 |
if (RedAlign > StackAlign) { |
0 |
| 2384 |
EVT IntermediateVT; |
0 |
2384 |
EVT IntermediateVT; |
0 |
| 2385 |
MVT RegisterVT; |
0 |
2385 |
MVT RegisterVT; |
0 |
| 2386 |
unsigned NumIntermediates; |
--- |
2386 |
unsigned NumIntermediates; |
--- |
| 2387 |
TLI->getVectorTypeBreakdown(*getContext(), VT, IntermediateVT, |
0 |
2387 |
TLI->getVectorTypeBreakdown(*getContext(), VT, IntermediateVT, |
0 |
| 2388 |
NumIntermediates, RegisterVT); |
--- |
2388 |
NumIntermediates, RegisterVT); |
--- |
| 2389 |
Ty = IntermediateVT.getTypeForEVT(*getContext()); |
0 |
2389 |
Ty = IntermediateVT.getTypeForEVT(*getContext()); |
0 |
| 2390 |
Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty); |
0 |
2390 |
Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty); |
0 |
| 2391 |
if (RedAlign2 < RedAlign) |
0 |
2391 |
if (RedAlign2 < RedAlign) |
0 |
| 2392 |
RedAlign = RedAlign2; |
0 |
2392 |
RedAlign = RedAlign2; |
0 |
| 2393 |
} |
--- |
2393 |
} |
--- |
| 2394 |
|
--- |
2394 |
|
--- |
| 2395 |
return RedAlign; |
0 |
2395 |
return RedAlign; |
0 |
| 2396 |
} |
--- |
2396 |
} |
--- |
| 2397 |
|
--- |
2397 |
|
--- |
| 2398 |
SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) { |
0 |
2398 |
SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) { |
0 |
| 2399 |
MachineFrameInfo &MFI = MF->getFrameInfo(); |
0 |
2399 |
MachineFrameInfo &MFI = MF->getFrameInfo(); |
0 |
| 2400 |
const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); |
0 |
2400 |
const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); |
0 |
| 2401 |
int StackID = 0; |
0 |
2401 |
int StackID = 0; |
0 |
| 2402 |
if (Bytes.isScalable()) |
0 |
2402 |
if (Bytes.isScalable()) |
0 |
| 2403 |
StackID = TFI->getStackIDForScalableVectors(); |
0 |
2403 |
StackID = TFI->getStackIDForScalableVectors(); |
0 |
| 2404 |
// The stack id gives an indication of whether the object is scalable or |
--- |
2404 |
// The stack id gives an indication of whether the object is scalable or |
--- |
| 2405 |
// not, so it's safe to pass in the minimum size here. |
--- |
2405 |
// not, so it's safe to pass in the minimum size here. |
--- |
| 2406 |
int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinValue(), Alignment, |
0 |
2406 |
int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinValue(), Alignment, |
0 |
| 2407 |
false, nullptr, StackID); |
--- |
2407 |
false, nullptr, StackID); |
--- |
| 2408 |
return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout())); |
0 |
2408 |
return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout())); |
0 |
| 2409 |
} |
--- |
2409 |
} |
--- |
| 2410 |
|
--- |
2410 |
|
--- |
| 2411 |
SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { |
0 |
2411 |
SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { |
0 |
| 2412 |
Type *Ty = VT.getTypeForEVT(*getContext()); |
0 |
2412 |
Type *Ty = VT.getTypeForEVT(*getContext()); |
0 |
| 2413 |
Align StackAlign = |
--- |
2413 |
Align StackAlign = |
--- |
| 2414 |
std::max(getDataLayout().getPrefTypeAlign(Ty), Align(minAlign)); |
0 |
2414 |
std::max(getDataLayout().getPrefTypeAlign(Ty), Align(minAlign)); |
0 |
| 2415 |
return CreateStackTemporary(VT.getStoreSize(), StackAlign); |
0 |
2415 |
return CreateStackTemporary(VT.getStoreSize(), StackAlign); |
0 |
| 2416 |
} |
--- |
2416 |
} |
--- |
| 2417 |
|
--- |
2417 |
|
--- |
| 2418 |
SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { |
0 |
2418 |
SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { |
0 |
| 2419 |
TypeSize VT1Size = VT1.getStoreSize(); |
0 |
2419 |
TypeSize VT1Size = VT1.getStoreSize(); |
0 |
| 2420 |
TypeSize VT2Size = VT2.getStoreSize(); |
0 |
2420 |
TypeSize VT2Size = VT2.getStoreSize(); |
0 |
| 2421 |
assert(VT1Size.isScalable() == VT2Size.isScalable() && |
0 |
2421 |
assert(VT1Size.isScalable() == VT2Size.isScalable() && |
0 |
| 2422 |
"Don't know how to choose the maximum size when creating a stack " |
--- |
2422 |
"Don't know how to choose the maximum size when creating a stack " |
--- |
| 2423 |
"temporary"); |
--- |
2423 |
"temporary"); |
--- |
| 2424 |
TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue() |
0 |
2424 |
TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue() |
0 |
| 2425 |
? VT1Size |
0 |
2425 |
? VT1Size |
0 |
| 2426 |
: VT2Size; |
0 |
2426 |
: VT2Size; |
0 |
| 2427 |
|
--- |
2427 |
|
--- |
| 2428 |
Type *Ty1 = VT1.getTypeForEVT(*getContext()); |
0 |
2428 |
Type *Ty1 = VT1.getTypeForEVT(*getContext()); |
0 |
| 2429 |
Type *Ty2 = VT2.getTypeForEVT(*getContext()); |
0 |
2429 |
Type *Ty2 = VT2.getTypeForEVT(*getContext()); |
0 |
| 2430 |
const DataLayout &DL = getDataLayout(); |
0 |
2430 |
const DataLayout &DL = getDataLayout(); |
0 |
| 2431 |
Align Align = std::max(DL.getPrefTypeAlign(Ty1), DL.getPrefTypeAlign(Ty2)); |
0 |
2431 |
Align Align = std::max(DL.getPrefTypeAlign(Ty1), DL.getPrefTypeAlign(Ty2)); |
0 |
| 2432 |
return CreateStackTemporary(Bytes, Align); |
0 |
2432 |
return CreateStackTemporary(Bytes, Align); |
0 |
| 2433 |
} |
--- |
2433 |
} |
--- |
| 2434 |
|
--- |
2434 |
|
--- |
| 2435 |
SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2, |
6 |
2435 |
SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2, |
6 |
| 2436 |
ISD::CondCode Cond, const SDLoc &dl) { |
--- |
2436 |
ISD::CondCode Cond, const SDLoc &dl) { |
--- |
| 2437 |
EVT OpVT = N1.getValueType(); |
6 |
2437 |
EVT OpVT = N1.getValueType(); |
6 |
| 2438 |
|
--- |
2438 |
|
--- |
| 2439 |
auto GetUndefBooleanConstant = [&]() { |
0 |
2439 |
auto GetUndefBooleanConstant = [&]() { |
0 |
| 2440 |
if (VT.getScalarType() == MVT::i1 || |
0 |
2440 |
if (VT.getScalarType() == MVT::i1 || |
0 |
| 2441 |
TLI->getBooleanContents(OpVT) == |
0 |
2441 |
TLI->getBooleanContents(OpVT) == |
0 |
| 2442 |
TargetLowering::UndefinedBooleanContent) |
--- |
2442 |
TargetLowering::UndefinedBooleanContent) |
--- |
| 2443 |
return getUNDEF(VT); |
0 |
2443 |
return getUNDEF(VT); |
0 |
| 2444 |
// ZeroOrOne / ZeroOrNegative require specific values for the high bits, |
--- |
2444 |
// ZeroOrOne / ZeroOrNegative require specific values for the high bits, |
--- |
| 2445 |
// so we cannot use getUNDEF(). Return zero instead. |
--- |
2445 |
// so we cannot use getUNDEF(). Return zero instead. |
--- |
| 2446 |
return getConstant(0, dl, VT); |
0 |
2446 |
return getConstant(0, dl, VT); |
0 |
| 2447 |
}; |
6 |
2447 |
}; |
6 |
| 2448 |
|
--- |
2448 |
|
--- |
| 2449 |
// These setcc operations always fold. |
--- |
2449 |
// These setcc operations always fold. |
--- |
| 2450 |
switch (Cond) { |
6 |
2450 |
switch (Cond) { |
6 |
| 2451 |
default: break; |
6 |
2451 |
default: break; |
6 |
| 2452 |
case ISD::SETFALSE: |
0 |
2452 |
case ISD::SETFALSE: |
0 |
| 2453 |
case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT); |
0 |
2453 |
case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT); |
0 |
| 2454 |
case ISD::SETTRUE: |
0 |
2454 |
case ISD::SETTRUE: |
0 |
| 2455 |
case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT); |
0 |
2455 |
case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT); |
0 |
| 2456 |
|
--- |
2456 |
|
--- |
| 2457 |
case ISD::SETOEQ: |
0 |
2457 |
case ISD::SETOEQ: |
0 |
| 2458 |
case ISD::SETOGT: |
--- |
2458 |
case ISD::SETOGT: |
--- |
| 2459 |
case ISD::SETOGE: |
--- |
2459 |
case ISD::SETOGE: |
--- |
| 2460 |
case ISD::SETOLT: |
--- |
2460 |
case ISD::SETOLT: |
--- |
| 2461 |
case ISD::SETOLE: |
--- |
2461 |
case ISD::SETOLE: |
--- |
| 2462 |
case ISD::SETONE: |
--- |
2462 |
case ISD::SETONE: |
--- |
| 2463 |
case ISD::SETO: |
--- |
2463 |
case ISD::SETO: |
--- |
| 2464 |
case ISD::SETUO: |
--- |
2464 |
case ISD::SETUO: |
--- |
| 2465 |
case ISD::SETUEQ: |
--- |
2465 |
case ISD::SETUEQ: |
--- |
| 2466 |
case ISD::SETUNE: |
--- |
2466 |
case ISD::SETUNE: |
--- |
| 2467 |
assert(!OpVT.isInteger() && "Illegal setcc for integer!"); |
0 |
2467 |
assert(!OpVT.isInteger() && "Illegal setcc for integer!"); |
0 |
| 2468 |
break; |
0 |
2468 |
break; |
0 |
| 2469 |
} |
--- |
2469 |
} |
--- |
| 2470 |
|
--- |
2470 |
|
--- |
| 2471 |
if (OpVT.isInteger()) { |
6 |
2471 |
if (OpVT.isInteger()) { |
6 |
| 2472 |
// For EQ and NE, we can always pick a value for the undef to make the |
--- |
2472 |
// For EQ and NE, we can always pick a value for the undef to make the |
--- |
| 2473 |
// predicate pass or fail, so we can return undef. |
--- |
2473 |
// predicate pass or fail, so we can return undef. |
--- |
| 2474 |
// Matches behavior in llvm::ConstantFoldCompareInstruction. |
--- |
2474 |
// Matches behavior in llvm::ConstantFoldCompareInstruction. |
--- |
| 2475 |
// icmp eq/ne X, undef -> undef. |
--- |
2475 |
// icmp eq/ne X, undef -> undef. |
--- |
| 2476 |
if ((N1.isUndef() || N2.isUndef()) && |
6 |
2476 |
if ((N1.isUndef() || N2.isUndef()) && |
6 |
| 2477 |
(Cond == ISD::SETEQ || Cond == ISD::SETNE)) |
0 |
2477 |
(Cond == ISD::SETEQ || Cond == ISD::SETNE)) |
0 |
| 2478 |
return GetUndefBooleanConstant(); |
0 |
2478 |
return GetUndefBooleanConstant(); |
0 |
| 2479 |
|
--- |
2479 |
|
--- |
| 2480 |
// If both operands are undef, we can return undef for int comparison. |
--- |
2480 |
// If both operands are undef, we can return undef for int comparison. |
--- |
| 2481 |
// icmp undef, undef -> undef. |
--- |
2481 |
// icmp undef, undef -> undef. |
--- |
| 2482 |
if (N1.isUndef() && N2.isUndef()) |
6 |
2482 |
if (N1.isUndef() && N2.isUndef()) |
6 |
| 2483 |
return GetUndefBooleanConstant(); |
0 |
2483 |
return GetUndefBooleanConstant(); |
0 |
| 2484 |
|
--- |
2484 |
|
--- |
| 2485 |
// icmp X, X -> true/false |
--- |
2485 |
// icmp X, X -> true/false |
--- |
| 2486 |
// icmp X, undef -> true/false because undef could be X. |
--- |
2486 |
// icmp X, undef -> true/false because undef could be X. |
--- |
| 2487 |
if (N1 == N2) |
6 |
2487 |
if (N1 == N2) |
6 |
| 2488 |
return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT); |
0 |
2488 |
return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT); |
0 |
| 2489 |
} |
--- |
2489 |
} |
--- |
| 2490 |
|
--- |
2490 |
|
--- |
| 2491 |
if (ConstantSDNode *N2C = dyn_cast(N2)) { |
6 |
2491 |
if (ConstantSDNode *N2C = dyn_cast(N2)) { |
6 |
| 2492 |
const APInt &C2 = N2C->getAPIntValue(); |
0 |
2492 |
const APInt &C2 = N2C->getAPIntValue(); |
0 |
| 2493 |
if (ConstantSDNode *N1C = dyn_cast(N1)) { |
0 |
2493 |
if (ConstantSDNode *N1C = dyn_cast(N1)) { |
0 |
| 2494 |
const APInt &C1 = N1C->getAPIntValue(); |
0 |
2494 |
const APInt &C1 = N1C->getAPIntValue(); |
0 |
| 2495 |
|
--- |
2495 |
|
--- |
| 2496 |
return getBoolConstant(ICmpInst::compare(C1, C2, getICmpCondCode(Cond)), |
0 |
2496 |
return getBoolConstant(ICmpInst::compare(C1, C2, getICmpCondCode(Cond)), |
0 |
| 2497 |
dl, VT, OpVT); |
0 |
2497 |
dl, VT, OpVT); |
0 |
| 2498 |
} |
--- |
2498 |
} |
--- |
| 2499 |
} |
--- |
2499 |
} |
--- |
| 2500 |
|
--- |
2500 |
|
--- |
| 2501 |
auto *N1CFP = dyn_cast(N1); |
6 |
2501 |
auto *N1CFP = dyn_cast(N1); |
6 |
| 2502 |
auto *N2CFP = dyn_cast(N2); |
6 |
2502 |
auto *N2CFP = dyn_cast(N2); |
6 |
| 2503 |
|
--- |
2503 |
|
--- |
| 2504 |
if (N1CFP && N2CFP) { |
6 |
2504 |
if (N1CFP && N2CFP) { |
6 |
| 2505 |
APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF()); |
0 |
2505 |
APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF()); |
0 |
| 2506 |
switch (Cond) { |
0 |
2506 |
switch (Cond) { |
0 |
| 2507 |
default: break; |
0 |
2507 |
default: break; |
0 |
| 2508 |
case ISD::SETEQ: if (R==APFloat::cmpUnordered) |
0 |
2508 |
case ISD::SETEQ: if (R==APFloat::cmpUnordered) |
0 |
| 2509 |
return GetUndefBooleanConstant(); |
0 |
2509 |
return GetUndefBooleanConstant(); |
0 |
| 2510 |
[[fallthrough]]; |
--- |
2510 |
[[fallthrough]]; |
--- |
| 2511 |
case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT, |
0 |
2511 |
case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT, |
0 |
| 2512 |
OpVT); |
0 |
2512 |
OpVT); |
0 |
| 2513 |
case ISD::SETNE: if (R==APFloat::cmpUnordered) |
0 |
2513 |
case ISD::SETNE: if (R==APFloat::cmpUnordered) |
0 |
| 2514 |
return GetUndefBooleanConstant(); |
0 |
2514 |
return GetUndefBooleanConstant(); |
0 |
| 2515 |
[[fallthrough]]; |
--- |
2515 |
[[fallthrough]]; |
--- |
| 2516 |
case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan || |
0 |
2516 |
case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan || |
0 |
| 2517 |
R==APFloat::cmpLessThan, dl, VT, |
--- |
2517 |
R==APFloat::cmpLessThan, dl, VT, |
--- |
| 2518 |
OpVT); |
0 |
2518 |
OpVT); |
0 |
| 2519 |
case ISD::SETLT: if (R==APFloat::cmpUnordered) |
0 |
2519 |
case ISD::SETLT: if (R==APFloat::cmpUnordered) |
0 |
| 2520 |
return GetUndefBooleanConstant(); |
0 |
2520 |
return GetUndefBooleanConstant(); |
0 |
| 2521 |
[[fallthrough]]; |
--- |
2521 |
[[fallthrough]]; |
--- |
| 2522 |
case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT, |
0 |
2522 |
case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT, |
0 |
| 2523 |
OpVT); |
0 |
2523 |
OpVT); |
0 |
| 2524 |
case ISD::SETGT: if (R==APFloat::cmpUnordered) |
0 |
2524 |
case ISD::SETGT: if (R==APFloat::cmpUnordered) |
0 |
| 2525 |
return GetUndefBooleanConstant(); |
0 |
2525 |
return GetUndefBooleanConstant(); |
0 |
| 2526 |
[[fallthrough]]; |
--- |
2526 |
[[fallthrough]]; |
--- |
| 2527 |
case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl, |
0 |
2527 |
case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl, |
0 |
| 2528 |
VT, OpVT); |
0 |
2528 |
VT, OpVT); |
0 |
| 2529 |
case ISD::SETLE: if (R==APFloat::cmpUnordered) |
0 |
2529 |
case ISD::SETLE: if (R==APFloat::cmpUnordered) |
0 |
| 2530 |
return GetUndefBooleanConstant(); |
0 |
2530 |
return GetUndefBooleanConstant(); |
0 |
| 2531 |
[[fallthrough]]; |
--- |
2531 |
[[fallthrough]]; |
--- |
| 2532 |
case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan || |
0 |
2532 |
case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan || |
0 |
| 2533 |
R==APFloat::cmpEqual, dl, VT, |
--- |
2533 |
R==APFloat::cmpEqual, dl, VT, |
--- |
| 2534 |
OpVT); |
0 |
2534 |
OpVT); |
0 |
| 2535 |
case ISD::SETGE: if (R==APFloat::cmpUnordered) |
0 |
2535 |
case ISD::SETGE: if (R==APFloat::cmpUnordered) |
0 |
| 2536 |
return GetUndefBooleanConstant(); |
0 |
2536 |
return GetUndefBooleanConstant(); |
0 |
| 2537 |
[[fallthrough]]; |
--- |
2537 |
[[fallthrough]]; |
--- |
| 2538 |
case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan || |
0 |
2538 |
case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan || |
0 |
| 2539 |
R==APFloat::cmpEqual, dl, VT, OpVT); |
0 |
2539 |
R==APFloat::cmpEqual, dl, VT, OpVT); |
0 |
| 2540 |
case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT, |
0 |
2540 |
case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT, |
0 |
| 2541 |
OpVT); |
0 |
2541 |
OpVT); |
0 |
| 2542 |
case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT, |
0 |
2542 |
case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT, |
0 |
| 2543 |
OpVT); |
0 |
2543 |
OpVT); |
0 |
| 2544 |
case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered || |
0 |
2544 |
case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered || |
0 |
| 2545 |
R==APFloat::cmpEqual, dl, VT, |
--- |
2545 |
R==APFloat::cmpEqual, dl, VT, |
--- |
| 2546 |
OpVT); |
0 |
2546 |
OpVT); |
0 |
| 2547 |
case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT, |
0 |
2547 |
case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT, |
0 |
| 2548 |
OpVT); |
0 |
2548 |
OpVT); |
0 |
| 2549 |
case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered || |
0 |
2549 |
case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered || |
0 |
| 2550 |
R==APFloat::cmpLessThan, dl, VT, |
--- |
2550 |
R==APFloat::cmpLessThan, dl, VT, |
--- |
| 2551 |
OpVT); |
0 |
2551 |
OpVT); |
0 |
| 2552 |
case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan || |
0 |
2552 |
case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan || |
0 |
| 2553 |
R==APFloat::cmpUnordered, dl, VT, |
--- |
2553 |
R==APFloat::cmpUnordered, dl, VT, |
--- |
| 2554 |
OpVT); |
0 |
2554 |
OpVT); |
0 |
| 2555 |
case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl, |
0 |
2555 |
case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl, |
0 |
| 2556 |
VT, OpVT); |
0 |
2556 |
VT, OpVT); |
0 |
| 2557 |
case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT, |
0 |
2557 |
case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT, |
0 |
| 2558 |
OpVT); |
0 |
2558 |
OpVT); |
0 |
| 2559 |
} |
--- |
2559 |
} |
--- |
| 2560 |
} else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) { |
6 |
2560 |
} else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) { |
6 |
| 2561 |
// Ensure that the constant occurs on the RHS. |
--- |
2561 |
// Ensure that the constant occurs on the RHS. |
--- |
| 2562 |
ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond); |
0 |
2562 |
ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond); |
0 |
| 2563 |
if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT())) |
0 |
2563 |
if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT())) |
0 |
| 2564 |
return SDValue(); |
0 |
2564 |
return SDValue(); |
0 |
| 2565 |
return getSetCC(dl, VT, N2, N1, SwappedCond); |
0 |
2565 |
return getSetCC(dl, VT, N2, N1, SwappedCond); |
0 |
| 2566 |
} else if ((N2CFP && N2CFP->getValueAPF().isNaN()) || |
12 |
2566 |
} else if ((N2CFP && N2CFP->getValueAPF().isNaN()) || |
12 |
| 2567 |
(OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) { |
6 |
2567 |
(OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) { |
6 |
| 2568 |
// If an operand is known to be a nan (or undef that could be a nan), we can |
--- |
2568 |
// If an operand is known to be a nan (or undef that could be a nan), we can |
--- |
| 2569 |
// fold it. |
--- |
2569 |
// fold it. |
--- |
| 2570 |
// Choosing NaN for the undef will always make unordered comparison succeed |
--- |
2570 |
// Choosing NaN for the undef will always make unordered comparison succeed |
--- |
| 2571 |
// and ordered comparison fails. |
--- |
2571 |
// and ordered comparison fails. |
--- |
| 2572 |
// Matches behavior in llvm::ConstantFoldCompareInstruction. |
--- |
2572 |
// Matches behavior in llvm::ConstantFoldCompareInstruction. |
--- |
| 2573 |
switch (ISD::getUnorderedFlavor(Cond)) { |
0 |
2573 |
switch (ISD::getUnorderedFlavor(Cond)) { |
0 |
| 2574 |
default: |
0 |
2574 |
default: |
0 |
| 2575 |
llvm_unreachable("Unknown flavor!"); |
0 |
2575 |
llvm_unreachable("Unknown flavor!"); |
0 |
| 2576 |
case 0: // Known false. |
0 |
2576 |
case 0: // Known false. |
0 |
| 2577 |
return getBoolConstant(false, dl, VT, OpVT); |
0 |
2577 |
return getBoolConstant(false, dl, VT, OpVT); |
0 |
| 2578 |
case 1: // Known true. |
0 |
2578 |
case 1: // Known true. |
0 |
| 2579 |
return getBoolConstant(true, dl, VT, OpVT); |
0 |
2579 |
return getBoolConstant(true, dl, VT, OpVT); |
0 |
| 2580 |
case 2: // Undefined. |
0 |
2580 |
case 2: // Undefined. |
0 |
| 2581 |
return GetUndefBooleanConstant(); |
0 |
2581 |
return GetUndefBooleanConstant(); |
0 |
| 2582 |
} |
--- |
2582 |
} |
--- |
| 2583 |
} |
--- |
2583 |
} |
--- |
| 2584 |
|
--- |
2584 |
|
--- |
| 2585 |
// Could not fold it. |
--- |
2585 |
// Could not fold it. |
--- |
| 2586 |
return SDValue(); |
6 |
2586 |
return SDValue(); |
6 |
| 2587 |
} |
--- |
2587 |
} |
--- |
| 2588 |
|
--- |
2588 |
|
--- |
| 2589 |
/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We |
--- |
2589 |
/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We |
--- |
| 2590 |
/// use this predicate to simplify operations downstream. |
--- |
2590 |
/// use this predicate to simplify operations downstream. |
--- |
| 2591 |
bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { |
0 |
2591 |
bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { |
0 |
| 2592 |
unsigned BitWidth = Op.getScalarValueSizeInBits(); |
0 |
2592 |
unsigned BitWidth = Op.getScalarValueSizeInBits(); |
0 |
| 2593 |
return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth); |
0 |
2593 |
return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth); |
0 |
| 2594 |
} |
--- |
2594 |
} |
--- |
| 2595 |
|
--- |
2595 |
|
--- |
| 2596 |
/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use |
--- |
2596 |
/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use |
--- |
| 2597 |
/// this predicate to simplify operations downstream. Mask is known to be zero |
--- |
2597 |
/// this predicate to simplify operations downstream. Mask is known to be zero |
--- |
| 2598 |
/// for bits that V cannot have. |
--- |
2598 |
/// for bits that V cannot have. |
--- |
| 2599 |
bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask, |
1 |
2599 |
bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask, |
1 |
| 2600 |
unsigned Depth) const { |
--- |
2600 |
unsigned Depth) const { |
--- |
| 2601 |
return Mask.isSubsetOf(computeKnownBits(V, Depth).Zero); |
1 |
2601 |
return Mask.isSubsetOf(computeKnownBits(V, Depth).Zero); |
1 |
| 2602 |
} |
--- |
2602 |
} |
--- |
| 2603 |
|
--- |
2603 |
|
--- |
| 2604 |
/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in |
--- |
2604 |
/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in |
--- |
| 2605 |
/// DemandedElts. We use this predicate to simplify operations downstream. |
--- |
2605 |
/// DemandedElts. We use this predicate to simplify operations downstream. |
--- |
| 2606 |
/// Mask is known to be zero for bits that V cannot have. |
--- |
2606 |
/// Mask is known to be zero for bits that V cannot have. |
--- |
| 2607 |
bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask, |
0 |
2607 |
bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask, |
0 |
| 2608 |
const APInt &DemandedElts, |
--- |
2608 |
const APInt &DemandedElts, |
--- |
| 2609 |
unsigned Depth) const { |
--- |
2609 |
unsigned Depth) const { |
--- |
| 2610 |
return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero); |
0 |
2610 |
return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero); |
0 |
| 2611 |
} |
--- |
2611 |
} |
--- |
| 2612 |
|
--- |
2612 |
|
--- |
| 2613 |
/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in |
--- |
2613 |
/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in |
--- |
| 2614 |
/// DemandedElts. We use this predicate to simplify operations downstream. |
--- |
2614 |
/// DemandedElts. We use this predicate to simplify operations downstream. |
--- |
| 2615 |
bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts, |
0 |
2615 |
bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts, |
0 |
| 2616 |
unsigned Depth /* = 0 */) const { |
--- |
2616 |
unsigned Depth /* = 0 */) const { |
--- |
| 2617 |
return computeKnownBits(V, DemandedElts, Depth).isZero(); |
0 |
2617 |
return computeKnownBits(V, DemandedElts, Depth).isZero(); |
0 |
| 2618 |
} |
--- |
2618 |
} |
--- |
| 2619 |
|
--- |
2619 |
|
--- |
| 2620 |
/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'. |
--- |
2620 |
/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'. |
--- |
| 2621 |
bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask, |
0 |
2621 |
bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask, |
0 |
| 2622 |
unsigned Depth) const { |
--- |
2622 |
unsigned Depth) const { |
--- |
| 2623 |
return Mask.isSubsetOf(computeKnownBits(V, Depth).One); |
0 |
2623 |
return Mask.isSubsetOf(computeKnownBits(V, Depth).One); |
0 |
| 2624 |
} |
--- |
2624 |
} |
--- |
| 2625 |
|
--- |
2625 |
|
--- |
| 2626 |
APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op, |
0 |
2626 |
APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op, |
0 |
| 2627 |
const APInt &DemandedElts, |
--- |
2627 |
const APInt &DemandedElts, |
--- |
| 2628 |
unsigned Depth) const { |
--- |
2628 |
unsigned Depth) const { |
--- |
| 2629 |
EVT VT = Op.getValueType(); |
0 |
2629 |
EVT VT = Op.getValueType(); |
0 |
| 2630 |
assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!"); |
0 |
2630 |
assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!"); |
0 |
| 2631 |
|
--- |
2631 |
|
--- |
| 2632 |
unsigned NumElts = VT.getVectorNumElements(); |
0 |
2632 |
unsigned NumElts = VT.getVectorNumElements(); |
0 |
| 2633 |
assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask."); |
0 |
2633 |
assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask."); |
0 |
| 2634 |
|
--- |
2634 |
|
--- |
| 2635 |
APInt KnownZeroElements = APInt::getZero(NumElts); |
0 |
2635 |
APInt KnownZeroElements = APInt::getZero(NumElts); |
0 |
| 2636 |
for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) { |
0 |
2636 |
for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) { |
0 |
| 2637 |
if (!DemandedElts[EltIdx]) |
0 |
2637 |
if (!DemandedElts[EltIdx]) |
0 |
| 2638 |
continue; // Don't query elements that are not demanded. |
0 |
2638 |
continue; // Don't query elements that are not demanded. |
0 |
| 2639 |
APInt Mask = APInt::getOneBitSet(NumElts, EltIdx); |
0 |
2639 |
APInt Mask = APInt::getOneBitSet(NumElts, EltIdx); |
0 |
| 2640 |
if (MaskedVectorIsZero(Op, Mask, Depth)) |
0 |
2640 |
if (MaskedVectorIsZero(Op, Mask, Depth)) |
0 |
| 2641 |
KnownZeroElements.setBit(EltIdx); |
0 |
2641 |
KnownZeroElements.setBit(EltIdx); |
0 |
| 2642 |
} |
0 |
2642 |
} |
0 |
| 2643 |
return KnownZeroElements; |
0 |
2643 |
return KnownZeroElements; |
0 |
| 2644 |
} |
--- |
2644 |
} |
--- |
| 2645 |
|
--- |
2645 |
|
--- |
| 2646 |
/// isSplatValue - Return true if the vector V has the same value |
--- |
2646 |
/// isSplatValue - Return true if the vector V has the same value |
--- |
| 2647 |
/// across all DemandedElts. For scalable vectors, we don't know the |
--- |
2647 |
/// across all DemandedElts. For scalable vectors, we don't know the |
--- |
| 2648 |
/// number of lanes at compile time. Instead, we use a 1 bit APInt |
--- |
2648 |
/// number of lanes at compile time. Instead, we use a 1 bit APInt |
--- |
| 2649 |
/// to represent a conservative value for all lanes; that is, that |
--- |
2649 |
/// to represent a conservative value for all lanes; that is, that |
--- |
| 2650 |
/// one bit value is implicitly splatted across all lanes. |
--- |
2650 |
/// one bit value is implicitly splatted across all lanes. |
--- |
| 2651 |
bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts, |
0 |
2651 |
bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts, |
0 |
| 2652 |
APInt &UndefElts, unsigned Depth) const { |
--- |
2652 |
APInt &UndefElts, unsigned Depth) const { |
--- |
| 2653 |
unsigned Opcode = V.getOpcode(); |
0 |
2653 |
unsigned Opcode = V.getOpcode(); |
0 |
| 2654 |
EVT VT = V.getValueType(); |
0 |
2654 |
EVT VT = V.getValueType(); |
0 |
| 2655 |
assert(VT.isVector() && "Vector type expected"); |
0 |
2655 |
assert(VT.isVector() && "Vector type expected"); |
0 |
| 2656 |
assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) && |
0 |
2656 |
assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) && |
0 |
| 2657 |
"scalable demanded bits are ignored"); |
--- |
2657 |
"scalable demanded bits are ignored"); |
--- |
| 2658 |
|
--- |
2658 |
|
--- |
| 2659 |
if (!DemandedElts) |
0 |
2659 |
if (!DemandedElts) |
0 |
| 2660 |
return false; // No demanded elts, better to assume we don't know anything. |
0 |
2660 |
return false; // No demanded elts, better to assume we don't know anything. |
0 |
| 2661 |
|
--- |
2661 |
|
--- |
| 2662 |
if (Depth >= MaxRecursionDepth) |
0 |
2662 |
if (Depth >= MaxRecursionDepth) |
0 |
| 2663 |
return false; // Limit search depth. |
0 |
2663 |
return false; // Limit search depth. |
0 |
| 2664 |
|
--- |
2664 |
|
--- |
| 2665 |
// Deal with some common cases here that work for both fixed and scalable |
--- |
2665 |
// Deal with some common cases here that work for both fixed and scalable |
--- |
| 2666 |
// vector types. |
--- |
2666 |
// vector types. |
--- |
| 2667 |
switch (Opcode) { |
0 |
2667 |
switch (Opcode) { |
0 |
| 2668 |
case ISD::SPLAT_VECTOR: |
0 |
2668 |
case ISD::SPLAT_VECTOR: |
0 |
| 2669 |
UndefElts = V.getOperand(0).isUndef() |
0 |
2669 |
UndefElts = V.getOperand(0).isUndef() |
0 |
| 2670 |
? APInt::getAllOnes(DemandedElts.getBitWidth()) |
0 |
2670 |
? APInt::getAllOnes(DemandedElts.getBitWidth()) |
0 |
| 2671 |
: APInt(DemandedElts.getBitWidth(), 0); |
0 |
2671 |
: APInt(DemandedElts.getBitWidth(), 0); |
0 |
| 2672 |
return true; |
0 |
2672 |
return true; |
0 |
| 2673 |
case ISD::ADD: |
0 |
2673 |
case ISD::ADD: |
0 |
| 2674 |
case ISD::SUB: |
--- |
2674 |
case ISD::SUB: |
--- |
| 2675 |
case ISD::AND: |
--- |
2675 |
case ISD::AND: |
--- |
| 2676 |
case ISD::XOR: |
--- |
2676 |
case ISD::XOR: |
--- |
| 2677 |
case ISD::OR: { |
--- |
2677 |
case ISD::OR: { |
--- |
| 2678 |
APInt UndefLHS, UndefRHS; |
0 |
2678 |
APInt UndefLHS, UndefRHS; |
0 |
| 2679 |
SDValue LHS = V.getOperand(0); |
0 |
2679 |
SDValue LHS = V.getOperand(0); |
0 |
| 2680 |
SDValue RHS = V.getOperand(1); |
0 |
2680 |
SDValue RHS = V.getOperand(1); |
0 |
| 2681 |
if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) && |
0 |
2681 |
if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) && |
0 |
| 2682 |
isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1)) { |
0 |
2682 |
isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1)) { |
0 |
| 2683 |
UndefElts = UndefLHS | UndefRHS; |
0 |
2683 |
UndefElts = UndefLHS | UndefRHS; |
0 |
| 2684 |
return true; |
0 |
2684 |
return true; |
0 |
| 2685 |
} |
--- |
2685 |
} |
--- |
| 2686 |
return false; |
0 |
2686 |
return false; |
0 |
| 2687 |
} |
0 |
2687 |
} |
0 |
| 2688 |
case ISD::ABS: |
0 |
2688 |
case ISD::ABS: |
0 |
| 2689 |
case ISD::TRUNCATE: |
--- |
2689 |
case ISD::TRUNCATE: |
--- |
| 2690 |
case ISD::SIGN_EXTEND: |
--- |
2690 |
case ISD::SIGN_EXTEND: |
--- |
| 2691 |
case ISD::ZERO_EXTEND: |
--- |
2691 |
case ISD::ZERO_EXTEND: |
--- |
| 2692 |
return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1); |
0 |
2692 |
return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1); |
0 |
| 2693 |
default: |
0 |
2693 |
default: |
0 |
| 2694 |
if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN || |
0 |
2694 |
if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN || |
0 |
| 2695 |
Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) |
0 |
2695 |
Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) |
0 |
| 2696 |
return TLI->isSplatValueForTargetNode(V, DemandedElts, UndefElts, *this, |
0 |
2696 |
return TLI->isSplatValueForTargetNode(V, DemandedElts, UndefElts, *this, |
0 |
| 2697 |
Depth); |
0 |
2697 |
Depth); |
0 |
| 2698 |
break; |
0 |
2698 |
break; |
0 |
| 2699 |
} |
--- |
2699 |
} |
--- |
| 2700 |
|
--- |
2700 |
|
--- |
| 2701 |
// We don't support other cases than those above for scalable vectors at |
--- |
2701 |
// We don't support other cases than those above for scalable vectors at |
--- |
| 2702 |
// the moment. |
--- |
2702 |
// the moment. |
--- |
| 2703 |
if (VT.isScalableVector()) |
0 |
2703 |
if (VT.isScalableVector()) |
0 |
| 2704 |
return false; |
0 |
2704 |
return false; |
0 |
| 2705 |
|
--- |
2705 |
|
--- |
| 2706 |
unsigned NumElts = VT.getVectorNumElements(); |
0 |
2706 |
unsigned NumElts = VT.getVectorNumElements(); |
0 |
| 2707 |
assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch"); |
0 |
2707 |
assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch"); |
0 |
| 2708 |
UndefElts = APInt::getZero(NumElts); |
0 |
2708 |
UndefElts = APInt::getZero(NumElts); |
0 |
| 2709 |
|
--- |
2709 |
|
--- |
| 2710 |
switch (Opcode) { |
0 |
2710 |
switch (Opcode) { |
0 |
| 2711 |
case ISD::BUILD_VECTOR: { |
0 |
2711 |
case ISD::BUILD_VECTOR: { |
0 |
| 2712 |
SDValue Scl; |
0 |
2712 |
SDValue Scl; |
0 |
| 2713 |
for (unsigned i = 0; i != NumElts; ++i) { |
0 |
2713 |
for (unsigned i = 0; i != NumElts; ++i) { |
0 |
| 2714 |
SDValue Op = V.getOperand(i); |
0 |
2714 |
SDValue Op = V.getOperand(i); |
0 |
| 2715 |
if (Op.isUndef()) { |
0 |
2715 |
if (Op.isUndef()) { |
0 |
| 2716 |
UndefElts.setBit(i); |
0 |
2716 |
UndefElts.setBit(i); |
0 |
| 2717 |
continue; |
0 |
2717 |
continue; |
0 |
| 2718 |
} |
--- |
2718 |
} |
--- |
| 2719 |
if (!DemandedElts[i]) |
0 |
2719 |
if (!DemandedElts[i]) |
0 |
| 2720 |
continue; |
0 |
2720 |
continue; |
0 |
| 2721 |
if (Scl && Scl != Op) |
0 |
2721 |
if (Scl && Scl != Op) |
0 |
| 2722 |
return false; |
0 |
2722 |
return false; |
0 |
| 2723 |
Scl = Op; |
0 |
2723 |
Scl = Op; |
0 |
| 2724 |
} |
--- |
2724 |
} |
--- |
| 2725 |
return true; |
0 |
2725 |
return true; |
0 |
| 2726 |
} |
--- |
2726 |
} |
--- |
| 2727 |
case ISD::VECTOR_SHUFFLE: { |
0 |
2727 |
case ISD::VECTOR_SHUFFLE: { |
0 |
| 2728 |
// Check if this is a shuffle node doing a splat or a shuffle of a splat. |
--- |
2728 |
// Check if this is a shuffle node doing a splat or a shuffle of a splat. |
--- |
| 2729 |
APInt DemandedLHS = APInt::getZero(NumElts); |
0 |
2729 |
APInt DemandedLHS = APInt::getZero(NumElts); |
0 |
| 2730 |
APInt DemandedRHS = APInt::getZero(NumElts); |
0 |
2730 |
APInt DemandedRHS = APInt::getZero(NumElts); |
0 |
| 2731 |
ArrayRef Mask = cast(V)->getMask(); |
0 |
2731 |
ArrayRef Mask = cast(V)->getMask(); |
0 |
| 2732 |
for (int i = 0; i != (int)NumElts; ++i) { |
0 |
2732 |
for (int i = 0; i != (int)NumElts; ++i) { |
0 |
| 2733 |
int M = Mask[i]; |
0 |
2733 |
int M = Mask[i]; |
0 |
| 2734 |
if (M < 0) { |
0 |
2734 |
if (M < 0) { |
0 |
| 2735 |
UndefElts.setBit(i); |
0 |
2735 |
UndefElts.setBit(i); |
0 |
| 2736 |
continue; |
0 |
2736 |
continue; |
0 |
| 2737 |
} |
--- |
2737 |
} |
--- |
| 2738 |
if (!DemandedElts[i]) |
0 |
2738 |
if (!DemandedElts[i]) |
0 |
| 2739 |
continue; |
0 |
2739 |
continue; |
0 |
| 2740 |
if (M < (int)NumElts) |
0 |
2740 |
if (M < (int)NumElts) |
0 |
| 2741 |
DemandedLHS.setBit(M); |
0 |
2741 |
DemandedLHS.setBit(M); |
0 |
| 2742 |
else |
--- |
2742 |
else |
--- |
| 2743 |
DemandedRHS.setBit(M - NumElts); |
0 |
2743 |
DemandedRHS.setBit(M - NumElts); |
0 |
| 2744 |
} |
--- |
2744 |
} |
--- |
| 2745 |
|
--- |
2745 |
|
--- |
| 2746 |
// If we aren't demanding either op, assume there's no splat. |
--- |
2746 |
// If we aren't demanding either op, assume there's no splat. |
--- |
| 2747 |
// If we are demanding both ops, assume there's no splat. |
--- |
2747 |
// If we are demanding both ops, assume there's no splat. |
--- |
| 2748 |
if ((DemandedLHS.isZero() && DemandedRHS.isZero()) || |
0 |
2748 |
if ((DemandedLHS.isZero() && DemandedRHS.isZero()) || |
0 |
| 2749 |
(!DemandedLHS.isZero() && !DemandedRHS.isZero())) |
0 |
2749 |
(!DemandedLHS.isZero() && !DemandedRHS.isZero())) |
0 |
| 2750 |
return false; |
0 |
2750 |
return false; |
0 |
| 2751 |
|
--- |
2751 |
|
--- |
| 2752 |
// See if the demanded elts of the source op is a splat or we only demand |
--- |
2752 |
// See if the demanded elts of the source op is a splat or we only demand |
--- |
| 2753 |
// one element, which should always be a splat. |
--- |
2753 |
// one element, which should always be a splat. |
--- |
| 2754 |
// TODO: Handle source ops splats with undefs. |
--- |
2754 |
// TODO: Handle source ops splats with undefs. |
--- |
| 2755 |
auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) { |
0 |
2755 |
auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) { |
0 |
| 2756 |
APInt SrcUndefs; |
0 |
2756 |
APInt SrcUndefs; |
0 |
| 2757 |
return (SrcElts.popcount() == 1) || |
0 |
2757 |
return (SrcElts.popcount() == 1) || |
0 |
| 2758 |
(isSplatValue(Src, SrcElts, SrcUndefs, Depth + 1) && |
0 |
2758 |
(isSplatValue(Src, SrcElts, SrcUndefs, Depth + 1) && |
0 |
| 2759 |
(SrcElts & SrcUndefs).isZero()); |
0 |
2759 |
(SrcElts & SrcUndefs).isZero()); |
0 |
| 2760 |
}; |
0 |
2760 |
}; |
0 |
| 2761 |
if (!DemandedLHS.isZero()) |
0 |
2761 |
if (!DemandedLHS.isZero()) |
0 |
| 2762 |
return CheckSplatSrc(V.getOperand(0), DemandedLHS); |
0 |
2762 |
return CheckSplatSrc(V.getOperand(0), DemandedLHS); |
0 |
| 2763 |
return CheckSplatSrc(V.getOperand(1), DemandedRHS); |
0 |
2763 |
return CheckSplatSrc(V.getOperand(1), DemandedRHS); |
0 |
| 2764 |
} |
0 |
2764 |
} |
0 |
| 2765 |
case ISD::EXTRACT_SUBVECTOR: { |
0 |
2765 |
case ISD::EXTRACT_SUBVECTOR: { |
0 |
| 2766 |
// Offset the demanded elts by the subvector index. |
--- |
2766 |
// Offset the demanded elts by the subvector index. |
--- |
| 2767 |
SDValue Src = V.getOperand(0); |
0 |
2767 |
SDValue Src = V.getOperand(0); |
0 |
| 2768 |
// We don't support scalable vectors at the moment. |
--- |
2768 |
// We don't support scalable vectors at the moment. |
--- |
| 2769 |
if (Src.getValueType().isScalableVector()) |
0 |
2769 |
if (Src.getValueType().isScalableVector()) |
0 |
| 2770 |
return false; |
0 |
2770 |
return false; |
0 |
| 2771 |
uint64_t Idx = V.getConstantOperandVal(1); |
0 |
2771 |
uint64_t Idx = V.getConstantOperandVal(1); |
0 |
| 2772 |
unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); |
0 |
2772 |
unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); |
0 |
| 2773 |
APInt UndefSrcElts; |
0 |
2773 |
APInt UndefSrcElts; |
0 |
| 2774 |
APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx); |
0 |
2774 |
APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx); |
0 |
| 2775 |
if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) { |
0 |
2775 |
if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) { |
0 |
| 2776 |
UndefElts = UndefSrcElts.extractBits(NumElts, Idx); |
0 |
2776 |
UndefElts = UndefSrcElts.extractBits(NumElts, Idx); |
0 |
| 2777 |
return true; |
0 |
2777 |
return true; |
0 |
| 2778 |
} |
--- |
2778 |
} |
--- |
| 2779 |
break; |
0 |
2779 |
break; |
0 |
| 2780 |
} |
0 |
2780 |
} |
0 |
| 2781 |
case ISD::ANY_EXTEND_VECTOR_INREG: |
0 |
2781 |
case ISD::ANY_EXTEND_VECTOR_INREG: |
0 |
| 2782 |
case ISD::SIGN_EXTEND_VECTOR_INREG: |
--- |
2782 |
case ISD::SIGN_EXTEND_VECTOR_INREG: |
--- |
| 2783 |
case ISD::ZERO_EXTEND_VECTOR_INREG: { |
--- |
2783 |
case ISD::ZERO_EXTEND_VECTOR_INREG: { |
--- |
| 2784 |
// Widen the demanded elts by the src element count. |
--- |
2784 |
// Widen the demanded elts by the src element count. |
--- |
| 2785 |
SDValue Src = V.getOperand(0); |
0 |
2785 |
SDValue Src = V.getOperand(0); |
0 |
| 2786 |
// We don't support scalable vectors at the moment. |
--- |
2786 |
// We don't support scalable vectors at the moment. |
--- |
| 2787 |
if (Src.getValueType().isScalableVector()) |
0 |
2787 |
if (Src.getValueType().isScalableVector()) |
0 |
| 2788 |
return false; |
0 |
2788 |
return false; |
0 |
| 2789 |
unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); |
0 |
2789 |
unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); |
0 |
| 2790 |
APInt UndefSrcElts; |
0 |
2790 |
APInt UndefSrcElts; |
0 |
| 2791 |
APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts); |
0 |
2791 |
APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts); |
0 |
| 2792 |
if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) { |
0 |
2792 |
if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) { |
0 |
| 2793 |
UndefElts = UndefSrcElts.trunc(NumElts); |
0 |
2793 |
UndefElts = UndefSrcElts.trunc(NumElts); |
0 |
| 2794 |
return true; |
0 |
2794 |
return true; |
0 |
| 2795 |
} |
--- |
2795 |
} |
--- |
| 2796 |
break; |
0 |
2796 |
break; |
0 |
| 2797 |
} |
0 |
2797 |
} |
0 |
| 2798 |
case ISD::BITCAST: { |
0 |
2798 |
case ISD::BITCAST: { |
0 |
| 2799 |
SDValue Src = V.getOperand(0); |
0 |
2799 |
SDValue Src = V.getOperand(0); |
0 |
| 2800 |
EVT SrcVT = Src.getValueType(); |
0 |
2800 |
EVT SrcVT = Src.getValueType(); |
0 |
| 2801 |
unsigned SrcBitWidth = SrcVT.getScalarSizeInBits(); |
0 |
2801 |
unsigned SrcBitWidth = SrcVT.getScalarSizeInBits(); |
0 |
| 2802 |
unsigned BitWidth = VT.getScalarSizeInBits(); |
0 |
2802 |
unsigned BitWidth = VT.getScalarSizeInBits(); |
0 |
| 2803 |
|
--- |
2803 |
|
--- |
| 2804 |
// Ignore bitcasts from unsupported types. |
--- |
2804 |
// Ignore bitcasts from unsupported types. |
--- |
| 2805 |
// TODO: Add fp support? |
--- |
2805 |
// TODO: Add fp support? |
--- |
| 2806 |
if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger()) |
0 |
2806 |
if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger()) |
0 |
| 2807 |
break; |
0 |
2807 |
break; |
0 |
| 2808 |
|
--- |
2808 |
|
--- |
| 2809 |
// Bitcast 'small element' vector to 'large element' vector. |
--- |
2809 |
// Bitcast 'small element' vector to 'large element' vector. |
--- |
| 2810 |
if ((BitWidth % SrcBitWidth) == 0) { |
0 |
2810 |
if ((BitWidth % SrcBitWidth) == 0) { |
0 |
| 2811 |
// See if each sub element is a splat. |
--- |
2811 |
// See if each sub element is a splat. |
--- |
| 2812 |
unsigned Scale = BitWidth / SrcBitWidth; |
0 |
2812 |
unsigned Scale = BitWidth / SrcBitWidth; |
0 |
| 2813 |
unsigned NumSrcElts = SrcVT.getVectorNumElements(); |
0 |
2813 |
unsigned NumSrcElts = SrcVT.getVectorNumElements(); |
0 |
| 2814 |
APInt ScaledDemandedElts = |
--- |
2814 |
APInt ScaledDemandedElts = |
--- |
| 2815 |
APIntOps::ScaleBitMask(DemandedElts, NumSrcElts); |
0 |
2815 |
APIntOps::ScaleBitMask(DemandedElts, NumSrcElts); |
0 |
| 2816 |
for (unsigned I = 0; I != Scale; ++I) { |
0 |
2816 |
for (unsigned I = 0; I != Scale; ++I) { |
0 |
| 2817 |
APInt SubUndefElts; |
0 |
2817 |
APInt SubUndefElts; |
0 |
| 2818 |
APInt SubDemandedElt = APInt::getOneBitSet(Scale, I); |
0 |
2818 |
APInt SubDemandedElt = APInt::getOneBitSet(Scale, I); |
0 |
| 2819 |
APInt SubDemandedElts = APInt::getSplat(NumSrcElts, SubDemandedElt); |
0 |
2819 |
APInt SubDemandedElts = APInt::getSplat(NumSrcElts, SubDemandedElt); |
0 |
| 2820 |
SubDemandedElts &= ScaledDemandedElts; |
0 |
2820 |
SubDemandedElts &= ScaledDemandedElts; |
0 |
| 2821 |
if (!isSplatValue(Src, SubDemandedElts, SubUndefElts, Depth + 1)) |
0 |
2821 |
if (!isSplatValue(Src, SubDemandedElts, SubUndefElts, Depth + 1)) |
0 |
| 2822 |
return false; |
0 |
2822 |
return false; |
0 |
| 2823 |
// TODO: Add support for merging sub undef elements. |
--- |
2823 |
// TODO: Add support for merging sub undef elements. |
--- |
| 2824 |
if (!SubUndefElts.isZero()) |
0 |
2824 |
if (!SubUndefElts.isZero()) |
0 |
| 2825 |
return false; |
0 |
2825 |
return false; |
0 |
| 2826 |
} |
0 |
2826 |
} |
0 |
| 2827 |
return true; |
0 |
2827 |
return true; |
0 |
| 2828 |
} |
0 |
2828 |
} |
0 |
| 2829 |
break; |
0 |
2829 |
break; |
0 |
| 2830 |
} |
--- |
2830 |
} |
--- |
| 2831 |
} |
--- |
2831 |
} |
--- |
| 2832 |
|
--- |
2832 |
|
--- |
| 2833 |
return false; |
0 |
2833 |
return false; |
0 |
| 2834 |
} |
--- |
2834 |
} |
--- |
| 2835 |
|
--- |
2835 |
|
--- |
| 2836 |
/// Helper wrapper to main isSplatValue function. |
--- |
2836 |
/// Helper wrapper to main isSplatValue function. |
--- |
| 2837 |
bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const { |
0 |
2837 |
bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const { |
0 |
| 2838 |
EVT VT = V.getValueType(); |
0 |
2838 |
EVT VT = V.getValueType(); |
0 |
| 2839 |
assert(VT.isVector() && "Vector type expected"); |
0 |
2839 |
assert(VT.isVector() && "Vector type expected"); |
0 |
| 2840 |
|
--- |
2840 |
|
--- |
| 2841 |
APInt UndefElts; |
0 |
2841 |
APInt UndefElts; |
0 |
| 2842 |
// Since the number of lanes in a scalable vector is unknown at compile time, |
--- |
2842 |
// Since the number of lanes in a scalable vector is unknown at compile time, |
--- |
| 2843 |
// we track one bit which is implicitly broadcast to all lanes. This means |
--- |
2843 |
// we track one bit which is implicitly broadcast to all lanes. This means |
--- |
| 2844 |
// that all lanes in a scalable vector are considered demanded. |
--- |
2844 |
// that all lanes in a scalable vector are considered demanded. |
--- |
| 2845 |
APInt DemandedElts |
--- |
2845 |
APInt DemandedElts |
--- |
| 2846 |
= APInt::getAllOnes(VT.isScalableVector() ? 1 : VT.getVectorNumElements()); |
0 |
2846 |
= APInt::getAllOnes(VT.isScalableVector() ? 1 : VT.getVectorNumElements()); |
0 |
| 2847 |
return isSplatValue(V, DemandedElts, UndefElts) && |
0 |
2847 |
return isSplatValue(V, DemandedElts, UndefElts) && |
0 |
| 2848 |
(AllowUndefs || !UndefElts); |
0 |
2848 |
(AllowUndefs || !UndefElts); |
0 |
| 2849 |
} |
0 |
2849 |
} |
0 |
| 2850 |
|
--- |
2850 |
|
--- |
| 2851 |
SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) { |
0 |
2851 |
SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) { |
0 |
| 2852 |
V = peekThroughExtractSubvectors(V); |
0 |
2852 |
V = peekThroughExtractSubvectors(V); |
0 |
| 2853 |
|
--- |
2853 |
|
--- |
| 2854 |
EVT VT = V.getValueType(); |
0 |
2854 |
EVT VT = V.getValueType(); |
0 |
| 2855 |
unsigned Opcode = V.getOpcode(); |
0 |
2855 |
unsigned Opcode = V.getOpcode(); |
0 |
| 2856 |
switch (Opcode) { |
0 |
2856 |
switch (Opcode) { |
0 |
| 2857 |
default: { |
0 |
2857 |
default: { |
0 |
| 2858 |
APInt UndefElts; |
0 |
2858 |
APInt UndefElts; |
0 |
| 2859 |
// Since the number of lanes in a scalable vector is unknown at compile time, |
--- |
2859 |
// Since the number of lanes in a scalable vector is unknown at compile time, |
--- |
| 2860 |
// we track one bit which is implicitly broadcast to all lanes. This means |
--- |
2860 |
// we track one bit which is implicitly broadcast to all lanes. This means |
--- |
| 2861 |
// that all lanes in a scalable vector are considered demanded. |
--- |
2861 |
// that all lanes in a scalable vector are considered demanded. |
--- |
| 2862 |
APInt DemandedElts |
--- |
2862 |
APInt DemandedElts |
--- |
| 2863 |
= APInt::getAllOnes(VT.isScalableVector() ? 1 : VT.getVectorNumElements()); |
0 |
2863 |
= APInt::getAllOnes(VT.isScalableVector() ? 1 : VT.getVectorNumElements()); |
0 |
| 2864 |
|
--- |
2864 |
|
--- |
| 2865 |
if (isSplatValue(V, DemandedElts, UndefElts)) { |
0 |
2865 |
if (isSplatValue(V, DemandedElts, UndefElts)) { |
0 |
| 2866 |
if (VT.isScalableVector()) { |
0 |
2866 |
if (VT.isScalableVector()) { |
0 |
| 2867 |
// DemandedElts and UndefElts are ignored for scalable vectors, since |
--- |
2867 |
// DemandedElts and UndefElts are ignored for scalable vectors, since |
--- |
| 2868 |
// the only supported cases are SPLAT_VECTOR nodes. |
--- |
2868 |
// the only supported cases are SPLAT_VECTOR nodes. |
--- |
| 2869 |
SplatIdx = 0; |
0 |
2869 |
SplatIdx = 0; |
0 |
| 2870 |
} else { |
--- |
2870 |
} else { |
--- |
| 2871 |
// Handle case where all demanded elements are UNDEF. |
--- |
2871 |
// Handle case where all demanded elements are UNDEF. |
--- |
| 2872 |
if (DemandedElts.isSubsetOf(UndefElts)) { |
0 |
2872 |
if (DemandedElts.isSubsetOf(UndefElts)) { |
0 |
| 2873 |
SplatIdx = 0; |
0 |
2873 |
SplatIdx = 0; |
0 |
| 2874 |
return getUNDEF(VT); |
0 |
2874 |
return getUNDEF(VT); |
0 |
| 2875 |
} |
--- |
2875 |
} |
--- |
| 2876 |
SplatIdx = (UndefElts & DemandedElts).countr_one(); |
0 |
2876 |
SplatIdx = (UndefElts & DemandedElts).countr_one(); |
0 |
| 2877 |
} |
--- |
2877 |
} |
--- |
| 2878 |
return V; |
0 |
2878 |
return V; |
0 |
| 2879 |
} |
--- |
2879 |
} |
--- |
| 2880 |
break; |
0 |
2880 |
break; |
0 |
| 2881 |
} |
0 |
2881 |
} |
0 |
| 2882 |
case ISD::SPLAT_VECTOR: |
0 |
2882 |
case ISD::SPLAT_VECTOR: |
0 |
| 2883 |
SplatIdx = 0; |
0 |
2883 |
SplatIdx = 0; |
0 |
| 2884 |
return V; |
0 |
2884 |
return V; |
0 |
| 2885 |
case ISD::VECTOR_SHUFFLE: { |
0 |
2885 |
case ISD::VECTOR_SHUFFLE: { |
0 |
| 2886 |
assert(!VT.isScalableVector()); |
0 |
2886 |
assert(!VT.isScalableVector()); |
0 |
| 2887 |
// Check if this is a shuffle node doing a splat. |
--- |
2887 |
// Check if this is a shuffle node doing a splat. |
--- |
| 2888 |
// TODO - remove this and rely purely on SelectionDAG::isSplatValue, |
--- |
2888 |
// TODO - remove this and rely purely on SelectionDAG::isSplatValue, |
--- |
| 2889 |
// getTargetVShiftNode currently struggles without the splat source. |
--- |
2889 |
// getTargetVShiftNode currently struggles without the splat source. |
--- |
| 2890 |
auto *SVN = cast(V); |
0 |
2890 |
auto *SVN = cast(V); |
0 |
| 2891 |
if (!SVN->isSplat()) |
0 |
2891 |
if (!SVN->isSplat()) |
0 |
| 2892 |
break; |
0 |
2892 |
break; |
0 |
| 2893 |
int Idx = SVN->getSplatIndex(); |
0 |
2893 |
int Idx = SVN->getSplatIndex(); |
0 |
| 2894 |
int NumElts = V.getValueType().getVectorNumElements(); |
0 |
2894 |
int NumElts = V.getValueType().getVectorNumElements(); |
0 |
| 2895 |
SplatIdx = Idx % NumElts; |
0 |
2895 |
SplatIdx = Idx % NumElts; |
0 |
| 2896 |
return V.getOperand(Idx / NumElts); |
0 |
2896 |
return V.getOperand(Idx / NumElts); |
0 |
| 2897 |
} |
--- |
2897 |
} |
--- |
| 2898 |
} |
--- |
2898 |
} |
--- |
| 2899 |
|
--- |
2899 |
|
--- |
| 2900 |
return SDValue(); |
0 |
2900 |
return SDValue(); |
0 |
| 2901 |
} |
--- |
2901 |
} |
--- |
| 2902 |
|
--- |
2902 |
|
--- |
| 2903 |
SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) { |
0 |
2903 |
SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) { |
0 |
| 2904 |
int SplatIdx; |
--- |
2904 |
int SplatIdx; |
--- |
| 2905 |
if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) { |
0 |
2905 |
if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) { |
0 |
| 2906 |
EVT SVT = SrcVector.getValueType().getScalarType(); |
0 |
2906 |
EVT SVT = SrcVector.getValueType().getScalarType(); |
0 |
| 2907 |
EVT LegalSVT = SVT; |
0 |
2907 |
EVT LegalSVT = SVT; |
0 |
| 2908 |
if (LegalTypes && !TLI->isTypeLegal(SVT)) { |
0 |
2908 |
if (LegalTypes && !TLI->isTypeLegal(SVT)) { |
0 |
| 2909 |
if (!SVT.isInteger()) |
0 |
2909 |
if (!SVT.isInteger()) |
0 |
| 2910 |
return SDValue(); |
0 |
2910 |
return SDValue(); |
0 |
| 2911 |
LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT); |
0 |
2911 |
LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT); |
0 |
| 2912 |
if (LegalSVT.bitsLT(SVT)) |
0 |
2912 |
if (LegalSVT.bitsLT(SVT)) |
0 |
| 2913 |
return SDValue(); |
0 |
2913 |
return SDValue(); |
0 |
| 2914 |
} |
--- |
2914 |
} |
--- |
| 2915 |
return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), LegalSVT, SrcVector, |
0 |
2915 |
return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), LegalSVT, SrcVector, |
0 |
| 2916 |
getVectorIdxConstant(SplatIdx, SDLoc(V))); |
0 |
2916 |
getVectorIdxConstant(SplatIdx, SDLoc(V))); |
0 |
| 2917 |
} |
--- |
2917 |
} |
--- |
| 2918 |
return SDValue(); |
0 |
2918 |
return SDValue(); |
0 |
| 2919 |
} |
--- |
2919 |
} |
--- |
| 2920 |
|
--- |
2920 |
|
--- |
| 2921 |
const APInt * |
--- |
2921 |
const APInt * |
--- |
| 2922 |
SelectionDAG::getValidShiftAmountConstant(SDValue V, |
0 |
2922 |
SelectionDAG::getValidShiftAmountConstant(SDValue V, |
0 |
| 2923 |
const APInt &DemandedElts) const { |
--- |
2923 |
const APInt &DemandedElts) const { |
--- |
| 2924 |
assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || |
0 |
2924 |
assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || |
0 |
| 2925 |
V.getOpcode() == ISD::SRA) && |
--- |
2925 |
V.getOpcode() == ISD::SRA) && |
--- |
| 2926 |
"Unknown shift node"); |
--- |
2926 |
"Unknown shift node"); |
--- |
| 2927 |
unsigned BitWidth = V.getScalarValueSizeInBits(); |
0 |
2927 |
unsigned BitWidth = V.getScalarValueSizeInBits(); |
0 |
| 2928 |
if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1), DemandedElts)) { |
0 |
2928 |
if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1), DemandedElts)) { |
0 |
| 2929 |
// Shifting more than the bitwidth is not valid. |
--- |
2929 |
// Shifting more than the bitwidth is not valid. |
--- |
| 2930 |
const APInt &ShAmt = SA->getAPIntValue(); |
0 |
2930 |
const APInt &ShAmt = SA->getAPIntValue(); |
0 |
| 2931 |
if (ShAmt.ult(BitWidth)) |
0 |
2931 |
if (ShAmt.ult(BitWidth)) |
0 |
| 2932 |
return &ShAmt; |
0 |
2932 |
return &ShAmt; |
0 |
| 2933 |
} |
--- |
2933 |
} |
--- |
| 2934 |
return nullptr; |
0 |
2934 |
return nullptr; |
0 |
| 2935 |
} |
--- |
2935 |
} |
--- |
| 2936 |
|
--- |
2936 |
|
--- |
| 2937 |
const APInt *SelectionDAG::getValidMinimumShiftAmountConstant( |
0 |
2937 |
const APInt *SelectionDAG::getValidMinimumShiftAmountConstant( |
0 |
| 2938 |
SDValue V, const APInt &DemandedElts) const { |
--- |
2938 |
SDValue V, const APInt &DemandedElts) const { |
--- |
| 2939 |
assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || |
0 |
2939 |
assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || |
0 |
| 2940 |
V.getOpcode() == ISD::SRA) && |
--- |
2940 |
V.getOpcode() == ISD::SRA) && |
--- |
| 2941 |
"Unknown shift node"); |
--- |
2941 |
"Unknown shift node"); |
--- |
| 2942 |
if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts)) |
0 |
2942 |
if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts)) |
0 |
| 2943 |
return ValidAmt; |
0 |
2943 |
return ValidAmt; |
0 |
| 2944 |
unsigned BitWidth = V.getScalarValueSizeInBits(); |
0 |
2944 |
unsigned BitWidth = V.getScalarValueSizeInBits(); |
0 |
| 2945 |
auto *BV = dyn_cast(V.getOperand(1)); |
0 |
2945 |
auto *BV = dyn_cast(V.getOperand(1)); |
0 |
| 2946 |
if (!BV) |
0 |
2946 |
if (!BV) |
0 |
| 2947 |
return nullptr; |
0 |
2947 |
return nullptr; |
0 |
| 2948 |
const APInt *MinShAmt = nullptr; |
0 |
2948 |
const APInt *MinShAmt = nullptr; |
0 |
| 2949 |
for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { |
0 |
2949 |
for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { |
0 |
| 2950 |
if (!DemandedElts[i]) |
0 |
2950 |
if (!DemandedElts[i]) |
0 |
| 2951 |
continue; |
0 |
2951 |
continue; |
0 |
| 2952 |
auto *SA = dyn_cast(BV->getOperand(i)); |
0 |
2952 |
auto *SA = dyn_cast(BV->getOperand(i)); |
0 |
| 2953 |
if (!SA) |
0 |
2953 |
if (!SA) |
0 |
| 2954 |
return nullptr; |
0 |
2954 |
return nullptr; |
0 |
| 2955 |
// Shifting more than the bitwidth is not valid. |
--- |
2955 |
// Shifting more than the bitwidth is not valid. |
--- |
| 2956 |
const APInt &ShAmt = SA->getAPIntValue(); |
0 |
2956 |
const APInt &ShAmt = SA->getAPIntValue(); |
0 |
| 2957 |
if (ShAmt.uge(BitWidth)) |
0 |
2957 |
if (ShAmt.uge(BitWidth)) |
0 |
| 2958 |
return nullptr; |
0 |
2958 |
return nullptr; |
0 |
| 2959 |
if (MinShAmt && MinShAmt->ule(ShAmt)) |
0 |
2959 |
if (MinShAmt && MinShAmt->ule(ShAmt)) |
0 |
| 2960 |
continue; |
0 |
2960 |
continue; |
0 |
| 2961 |
MinShAmt = &ShAmt; |
0 |
2961 |
MinShAmt = &ShAmt; |
0 |
| 2962 |
} |
--- |
2962 |
} |
--- |
| 2963 |
return MinShAmt; |
0 |
2963 |
return MinShAmt; |
0 |
| 2964 |
} |
--- |
2964 |
} |
--- |
| 2965 |
|
--- |
2965 |
|
--- |
| 2966 |
const APInt *SelectionDAG::getValidMaximumShiftAmountConstant( |
0 |
2966 |
const APInt *SelectionDAG::getValidMaximumShiftAmountConstant( |
0 |
| 2967 |
SDValue V, const APInt &DemandedElts) const { |
--- |
2967 |
SDValue V, const APInt &DemandedElts) const { |
--- |
| 2968 |
assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || |
0 |
2968 |
assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || |
0 |
| 2969 |
V.getOpcode() == ISD::SRA) && |
--- |
2969 |
V.getOpcode() == ISD::SRA) && |
--- |
| 2970 |
"Unknown shift node"); |
--- |
2970 |
"Unknown shift node"); |
--- |
| 2971 |
if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts)) |
0 |
2971 |
if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts)) |
0 |
| 2972 |
return ValidAmt; |
0 |
2972 |
return ValidAmt; |
0 |
| 2973 |
unsigned BitWidth = V.getScalarValueSizeInBits(); |
0 |
2973 |
unsigned BitWidth = V.getScalarValueSizeInBits(); |
0 |
| 2974 |
auto *BV = dyn_cast(V.getOperand(1)); |
0 |
2974 |
auto *BV = dyn_cast(V.getOperand(1)); |
0 |
| 2975 |
if (!BV) |
0 |
2975 |
if (!BV) |
0 |
| 2976 |
return nullptr; |
0 |
2976 |
return nullptr; |
0 |
| 2977 |
const APInt *MaxShAmt = nullptr; |
0 |
2977 |
const APInt *MaxShAmt = nullptr; |
0 |
| 2978 |
for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { |
0 |
2978 |
for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { |
0 |
| 2979 |
if (!DemandedElts[i]) |
0 |
2979 |
if (!DemandedElts[i]) |
0 |
| 2980 |
continue; |
0 |
2980 |
continue; |
0 |
| 2981 |
auto *SA = dyn_cast(BV->getOperand(i)); |
0 |
2981 |
auto *SA = dyn_cast(BV->getOperand(i)); |
0 |
| 2982 |
if (!SA) |
0 |
2982 |
if (!SA) |
0 |
| 2983 |
return nullptr; |
0 |
2983 |
return nullptr; |
0 |
| 2984 |
// Shifting more than the bitwidth is not valid. |
--- |
2984 |
// Shifting more than the bitwidth is not valid. |
--- |
| 2985 |
const APInt &ShAmt = SA->getAPIntValue(); |
0 |
2985 |
const APInt &ShAmt = SA->getAPIntValue(); |
0 |
| 2986 |
if (ShAmt.uge(BitWidth)) |
0 |
2986 |
if (ShAmt.uge(BitWidth)) |
0 |
| 2987 |
return nullptr; |
0 |
2987 |
return nullptr; |
0 |
| 2988 |
if (MaxShAmt && MaxShAmt->uge(ShAmt)) |
0 |
2988 |
if (MaxShAmt && MaxShAmt->uge(ShAmt)) |
0 |
| 2989 |
continue; |
0 |
2989 |
continue; |
0 |
| 2990 |
MaxShAmt = &ShAmt; |
0 |
2990 |
MaxShAmt = &ShAmt; |
0 |
| 2991 |
} |
--- |
2991 |
} |
--- |
| 2992 |
return MaxShAmt; |
0 |
2992 |
return MaxShAmt; |
0 |
| 2993 |
} |
--- |
2993 |
} |
--- |
| 2994 |
|
--- |
2994 |
|
--- |
| 2995 |
/// Determine which bits of Op are known to be either zero or one and return |
--- |
2995 |
/// Determine which bits of Op are known to be either zero or one and return |
--- |
| 2996 |
/// them in Known. For vectors, the known bits are those that are shared by |
--- |
2996 |
/// them in Known. For vectors, the known bits are those that are shared by |
--- |
| 2997 |
/// every vector element. |
--- |
2997 |
/// every vector element. |
--- |
| 2998 |
KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const { |
3 |
2998 |
KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const { |
3 |
| 2999 |
EVT VT = Op.getValueType(); |
3 |
2999 |
EVT VT = Op.getValueType(); |
3 |
| 3000 |
|
--- |
3000 |
|
--- |
| 3001 |
// Since the number of lanes in a scalable vector is unknown at compile time, |
--- |
3001 |
// Since the number of lanes in a scalable vector is unknown at compile time, |
--- |
| 3002 |
// we track one bit which is implicitly broadcast to all lanes. This means |
--- |
3002 |
// we track one bit which is implicitly broadcast to all lanes. This means |
--- |
| 3003 |
// that all lanes in a scalable vector are considered demanded. |
--- |
3003 |
// that all lanes in a scalable vector are considered demanded. |
--- |
| 3004 |
APInt DemandedElts = VT.isFixedLengthVector() |
3 |
3004 |
APInt DemandedElts = VT.isFixedLengthVector() |
3 |
| 3005 |
? APInt::getAllOnes(VT.getVectorNumElements()) |
--- |
3005 |
? APInt::getAllOnes(VT.getVectorNumElements()) |
--- |
| 3006 |
: APInt(1, 1); |
3 |
3006 |
: APInt(1, 1); |
3 |
| 3007 |
return computeKnownBits(Op, DemandedElts, Depth); |
3 |
3007 |
return computeKnownBits(Op, DemandedElts, Depth); |
3 |
| 3008 |
} |
3 |
3008 |
} |
3 |
| 3009 |
|
--- |
3009 |
|
--- |
| 3010 |
/// Determine which bits of Op are known to be either zero or one and return |
--- |
3010 |
/// Determine which bits of Op are known to be either zero or one and return |
--- |
| 3011 |
/// them in Known. The DemandedElts argument allows us to only collect the known |
--- |
3011 |
/// them in Known. The DemandedElts argument allows us to only collect the known |
--- |
| 3012 |
/// bits that are shared by the requested vector elements. |
--- |
3012 |
/// bits that are shared by the requested vector elements. |
--- |
| 3013 |
KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, |
6 |
3013 |
KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, |
6 |
| 3014 |
unsigned Depth) const { |
--- |
3014 |
unsigned Depth) const { |
--- |
| 3015 |
unsigned BitWidth = Op.getScalarValueSizeInBits(); |
6 |
3015 |
unsigned BitWidth = Op.getScalarValueSizeInBits(); |
6 |
| 3016 |
|
--- |
3016 |
|
--- |
| 3017 |
KnownBits Known(BitWidth); // Don't know anything. |
6 |
3017 |
KnownBits Known(BitWidth); // Don't know anything. |
6 |
| 3018 |
|
--- |
3018 |
|
--- |
| 3019 |
if (auto *C = dyn_cast(Op)) { |
6 |
3019 |
if (auto *C = dyn_cast(Op)) { |
6 |
| 3020 |
// We know all of the bits for a constant! |
--- |
3020 |
// We know all of the bits for a constant! |
--- |
| 3021 |
return KnownBits::makeConstant(C->getAPIntValue()); |
2 |
3021 |
return KnownBits::makeConstant(C->getAPIntValue()); |
2 |
| 3022 |
} |
--- |
3022 |
} |
--- |
| 3023 |
if (auto *C = dyn_cast(Op)) { |
4 |
3023 |
if (auto *C = dyn_cast(Op)) { |
4 |
| 3024 |
// We know all of the bits for a constant fp! |
--- |
3024 |
// We know all of the bits for a constant fp! |
--- |
| 3025 |
return KnownBits::makeConstant(C->getValueAPF().bitcastToAPInt()); |
0 |
3025 |
return KnownBits::makeConstant(C->getValueAPF().bitcastToAPInt()); |
0 |
| 3026 |
} |
--- |
3026 |
} |
--- |
| 3027 |
|
--- |
3027 |
|
--- |
| 3028 |
if (Depth >= MaxRecursionDepth) |
4 |
3028 |
if (Depth >= MaxRecursionDepth) |
4 |
| 3029 |
return Known; // Limit search depth. |
0 |
3029 |
return Known; // Limit search depth. |
0 |
| 3030 |
|
--- |
3030 |
|
--- |
| 3031 |
KnownBits Known2; |
4 |
3031 |
KnownBits Known2; |
4 |
| 3032 |
unsigned NumElts = DemandedElts.getBitWidth(); |
4 |
3032 |
unsigned NumElts = DemandedElts.getBitWidth(); |
4 |
| 3033 |
assert((!Op.getValueType().isFixedLengthVector() || |
4 |
3033 |
assert((!Op.getValueType().isFixedLengthVector() || |
4 |
| 3034 |
NumElts == Op.getValueType().getVectorNumElements()) && |
--- |
3034 |
NumElts == Op.getValueType().getVectorNumElements()) && |
--- |
| 3035 |
"Unexpected vector size"); |
--- |
3035 |
"Unexpected vector size"); |
--- |
| 3036 |
|
--- |
3036 |
|
--- |
| 3037 |
if (!DemandedElts) |
4 |
3037 |
if (!DemandedElts) |
4 |
| 3038 |
return Known; // No demanded elts, better to assume we don't know anything. |
0 |
3038 |
return Known; // No demanded elts, better to assume we don't know anything. |
0 |
| 3039 |
|
--- |
3039 |
|
--- |
| 3040 |
unsigned Opcode = Op.getOpcode(); |
4 |
3040 |
unsigned Opcode = Op.getOpcode(); |
4 |
| 3041 |
switch (Opcode) { |
4 |
3041 |
switch (Opcode) { |
4 |
| 3042 |
case ISD::MERGE_VALUES: |
0 |
3042 |
case ISD::MERGE_VALUES: |
0 |
| 3043 |
return computeKnownBits(Op.getOperand(Op.getResNo()), DemandedElts, |
0 |
3043 |
return computeKnownBits(Op.getOperand(Op.getResNo()), DemandedElts, |
0 |
| 3044 |
Depth + 1); |
0 |
3044 |
Depth + 1); |
0 |
| 3045 |
case ISD::SPLAT_VECTOR: { |
0 |
3045 |
case ISD::SPLAT_VECTOR: { |
0 |
| 3046 |
SDValue SrcOp = Op.getOperand(0); |
0 |
3046 |
SDValue SrcOp = Op.getOperand(0); |
0 |
| 3047 |
assert(SrcOp.getValueSizeInBits() >= BitWidth && |
0 |
3047 |
assert(SrcOp.getValueSizeInBits() >= BitWidth && |
0 |
| 3048 |
"Expected SPLAT_VECTOR implicit truncation"); |
--- |
3048 |
"Expected SPLAT_VECTOR implicit truncation"); |
--- |
| 3049 |
// Implicitly truncate the bits to match the official semantics of |
--- |
3049 |
// Implicitly truncate the bits to match the official semantics of |
--- |
| 3050 |
// SPLAT_VECTOR. |
--- |
3050 |
// SPLAT_VECTOR. |
--- |
| 3051 |
Known = computeKnownBits(SrcOp, Depth + 1).trunc(BitWidth); |
0 |
3051 |
Known = computeKnownBits(SrcOp, Depth + 1).trunc(BitWidth); |
0 |
| 3052 |
break; |
0 |
3052 |
break; |
0 |
| 3053 |
} |
--- |
3053 |
} |
--- |
| 3054 |
case ISD::BUILD_VECTOR: |
0 |
3054 |
case ISD::BUILD_VECTOR: |
0 |
| 3055 |
assert(!Op.getValueType().isScalableVector()); |
0 |
3055 |
assert(!Op.getValueType().isScalableVector()); |
0 |
| 3056 |
// Collect the known bits that are shared by every demanded vector element. |
--- |
3056 |
// Collect the known bits that are shared by every demanded vector element. |
--- |
| 3057 |
Known.Zero.setAllBits(); Known.One.setAllBits(); |
0 |
3057 |
Known.Zero.setAllBits(); Known.One.setAllBits(); |
0 |
| 3058 |
for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) { |
0 |
3058 |
for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) { |
0 |
| 3059 |
if (!DemandedElts[i]) |
0 |
3059 |
if (!DemandedElts[i]) |
0 |
| 3060 |
continue; |
0 |
3060 |
continue; |
0 |
| 3061 |
|
--- |
3061 |
|
--- |
| 3062 |
SDValue SrcOp = Op.getOperand(i); |
0 |
3062 |
SDValue SrcOp = Op.getOperand(i); |
0 |
| 3063 |
Known2 = computeKnownBits(SrcOp, Depth + 1); |
0 |
3063 |
Known2 = computeKnownBits(SrcOp, Depth + 1); |
0 |
| 3064 |
|
--- |
3064 |
|
--- |
| 3065 |
// BUILD_VECTOR can implicitly truncate sources, we must handle this. |
--- |
3065 |
// BUILD_VECTOR can implicitly truncate sources, we must handle this. |
--- |
| 3066 |
if (SrcOp.getValueSizeInBits() != BitWidth) { |
0 |
3066 |
if (SrcOp.getValueSizeInBits() != BitWidth) { |
0 |
| 3067 |
assert(SrcOp.getValueSizeInBits() > BitWidth && |
0 |
3067 |
assert(SrcOp.getValueSizeInBits() > BitWidth && |
0 |
| 3068 |
"Expected BUILD_VECTOR implicit truncation"); |
--- |
3068 |
"Expected BUILD_VECTOR implicit truncation"); |
--- |
| 3069 |
Known2 = Known2.trunc(BitWidth); |
0 |
3069 |
Known2 = Known2.trunc(BitWidth); |
0 |
| 3070 |
} |
--- |
3070 |
} |
--- |
| 3071 |
|
--- |
3071 |
|
--- |
| 3072 |
// Known bits are the values that are shared by every demanded element. |
--- |
3072 |
// Known bits are the values that are shared by every demanded element. |
--- |
| 3073 |
Known = Known.intersectWith(Known2); |
0 |
3073 |
Known = Known.intersectWith(Known2); |
0 |
| 3074 |
|
--- |
3074 |
|
--- |
| 3075 |
// If we don't know any bits, early out. |
--- |
3075 |
// If we don't know any bits, early out. |
--- |
| 3076 |
if (Known.isUnknown()) |
0 |
3076 |
if (Known.isUnknown()) |
0 |
| 3077 |
break; |
0 |
3077 |
break; |
0 |
| 3078 |
} |
--- |
3078 |
} |
--- |
| 3079 |
break; |
0 |
3079 |
break; |
0 |
| 3080 |
case ISD::VECTOR_SHUFFLE: { |
0 |
3080 |
case ISD::VECTOR_SHUFFLE: { |
0 |
| 3081 |
assert(!Op.getValueType().isScalableVector()); |
0 |
3081 |
assert(!Op.getValueType().isScalableVector()); |
0 |
| 3082 |
// Collect the known bits that are shared by every vector element referenced |
--- |
3082 |
// Collect the known bits that are shared by every vector element referenced |
--- |
| 3083 |
// by the shuffle. |
--- |
3083 |
// by the shuffle. |
--- |
| 3084 |
APInt DemandedLHS, DemandedRHS; |
0 |
3084 |
APInt DemandedLHS, DemandedRHS; |
0 |
| 3085 |
const ShuffleVectorSDNode *SVN = cast(Op); |
0 |
3085 |
const ShuffleVectorSDNode *SVN = cast(Op); |
0 |
| 3086 |
assert(NumElts == SVN->getMask().size() && "Unexpected vector size"); |
0 |
3086 |
assert(NumElts == SVN->getMask().size() && "Unexpected vector size"); |
0 |
| 3087 |
if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts, |
0 |
3087 |
if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts, |
0 |
| 3088 |
DemandedLHS, DemandedRHS)) |
--- |
3088 |
DemandedLHS, DemandedRHS)) |
--- |
| 3089 |
break; |
0 |
3089 |
break; |
0 |
| 3090 |
|
--- |
3090 |
|
--- |
| 3091 |
// Known bits are the values that are shared by every demanded element. |
--- |
3091 |
// Known bits are the values that are shared by every demanded element. |
--- |
| 3092 |
Known.Zero.setAllBits(); Known.One.setAllBits(); |
0 |
3092 |
Known.Zero.setAllBits(); Known.One.setAllBits(); |
0 |
| 3093 |
if (!!DemandedLHS) { |
0 |
3093 |
if (!!DemandedLHS) { |
0 |
| 3094 |
SDValue LHS = Op.getOperand(0); |
0 |
3094 |
SDValue LHS = Op.getOperand(0); |
0 |
| 3095 |
Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1); |
0 |
3095 |
Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1); |
0 |
| 3096 |
Known = Known.intersectWith(Known2); |
0 |
3096 |
Known = Known.intersectWith(Known2); |
0 |
| 3097 |
} |
--- |
3097 |
} |
--- |
| 3098 |
// If we don't know any bits, early out. |
--- |
3098 |
// If we don't know any bits, early out. |
--- |
| 3099 |
if (Known.isUnknown()) |
0 |
3099 |
if (Known.isUnknown()) |
0 |
| 3100 |
break; |
0 |
3100 |
break; |
0 |
| 3101 |
if (!!DemandedRHS) { |
0 |
3101 |
if (!!DemandedRHS) { |
0 |
| 3102 |
SDValue RHS = Op.getOperand(1); |
0 |
3102 |
SDValue RHS = Op.getOperand(1); |
0 |
| 3103 |
Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1); |
0 |
3103 |
Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1); |
0 |
| 3104 |
Known = Known.intersectWith(Known2); |
0 |
3104 |
Known = Known.intersectWith(Known2); |
0 |
| 3105 |
} |
--- |
3105 |
} |
--- |
| 3106 |
break; |
0 |
3106 |
break; |
0 |
| 3107 |
} |
0 |
3107 |
} |
0 |
| 3108 |
case ISD::VSCALE: { |
0 |
3108 |
case ISD::VSCALE: { |
0 |
| 3109 |
const Function &F = getMachineFunction().getFunction(); |
0 |
3109 |
const Function &F = getMachineFunction().getFunction(); |
0 |
| 3110 |
const APInt &Multiplier = Op.getConstantOperandAPInt(0); |
0 |
3110 |
const APInt &Multiplier = Op.getConstantOperandAPInt(0); |
0 |
| 3111 |
Known = getVScaleRange(&F, BitWidth).multiply(Multiplier).toKnownBits(); |
0 |
3111 |
Known = getVScaleRange(&F, BitWidth).multiply(Multiplier).toKnownBits(); |
0 |
| 3112 |
break; |
0 |
3112 |
break; |
0 |
| 3113 |
} |
--- |
3113 |
} |
--- |
| 3114 |
case ISD::CONCAT_VECTORS: { |
0 |
3114 |
case ISD::CONCAT_VECTORS: { |
0 |
| 3115 |
if (Op.getValueType().isScalableVector()) |
0 |
3115 |
if (Op.getValueType().isScalableVector()) |
0 |
| 3116 |
break; |
0 |
3116 |
break; |
0 |
| 3117 |
// Split DemandedElts and test each of the demanded subvectors. |
--- |
3117 |
// Split DemandedElts and test each of the demanded subvectors. |
--- |
| 3118 |
Known.Zero.setAllBits(); Known.One.setAllBits(); |
0 |
3118 |
Known.Zero.setAllBits(); Known.One.setAllBits(); |
0 |
| 3119 |
EVT SubVectorVT = Op.getOperand(0).getValueType(); |
0 |
3119 |
EVT SubVectorVT = Op.getOperand(0).getValueType(); |
0 |
| 3120 |
unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements(); |
0 |
3120 |
unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements(); |
0 |
| 3121 |
unsigned NumSubVectors = Op.getNumOperands(); |
0 |
3121 |
unsigned NumSubVectors = Op.getNumOperands(); |
0 |
| 3122 |
for (unsigned i = 0; i != NumSubVectors; ++i) { |
0 |
3122 |
for (unsigned i = 0; i != NumSubVectors; ++i) { |
0 |
| 3123 |
APInt DemandedSub = |
--- |
3123 |
APInt DemandedSub = |
--- |
| 3124 |
DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts); |
0 |
3124 |
DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts); |
0 |
| 3125 |
if (!!DemandedSub) { |
0 |
3125 |
if (!!DemandedSub) { |
0 |
| 3126 |
SDValue Sub = Op.getOperand(i); |
0 |
3126 |
SDValue Sub = Op.getOperand(i); |
0 |
| 3127 |
Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1); |
0 |
3127 |
Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1); |
0 |
| 3128 |
Known = Known.intersectWith(Known2); |
0 |
3128 |
Known = Known.intersectWith(Known2); |
0 |
| 3129 |
} |
--- |
3129 |
} |
--- |
| 3130 |
// If we don't know any bits, early out. |
--- |
3130 |
// If we don't know any bits, early out. |
--- |
| 3131 |
if (Known.isUnknown()) |
0 |
3131 |
if (Known.isUnknown()) |
0 |
| 3132 |
break; |
0 |
3132 |
break; |
0 |
| 3133 |
} |
0 |
3133 |
} |
0 |
| 3134 |
break; |
0 |
3134 |
break; |
0 |
| 3135 |
} |
--- |
3135 |
} |
--- |
| 3136 |
case ISD::INSERT_SUBVECTOR: { |
0 |
3136 |
case ISD::INSERT_SUBVECTOR: { |
0 |
| 3137 |
if (Op.getValueType().isScalableVector()) |
0 |
3137 |
if (Op.getValueType().isScalableVector()) |
0 |
| 3138 |
break; |
0 |
3138 |
break; |
0 |
| 3139 |
// Demand any elements from the subvector and the remainder from the src its |
--- |
3139 |
// Demand any elements from the subvector and the remainder from the src its |
--- |
| 3140 |
// inserted into. |
--- |
3140 |
// inserted into. |
--- |
| 3141 |
SDValue Src = Op.getOperand(0); |
0 |
3141 |
SDValue Src = Op.getOperand(0); |
0 |
| 3142 |
SDValue Sub = Op.getOperand(1); |
0 |
3142 |
SDValue Sub = Op.getOperand(1); |
0 |
| 3143 |
uint64_t Idx = Op.getConstantOperandVal(2); |
0 |
3143 |
uint64_t Idx = Op.getConstantOperandVal(2); |
0 |
| 3144 |
unsigned NumSubElts = Sub.getValueType().getVectorNumElements(); |
0 |
3144 |
unsigned NumSubElts = Sub.getValueType().getVectorNumElements(); |
0 |
| 3145 |
APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx); |
0 |
3145 |
APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx); |
0 |
| 3146 |
APInt DemandedSrcElts = DemandedElts; |
0 |
3146 |
APInt DemandedSrcElts = DemandedElts; |
0 |
| 3147 |
DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx); |
0 |
3147 |
DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx); |
0 |
| 3148 |
|
--- |
3148 |
|
--- |
| 3149 |
Known.One.setAllBits(); |
0 |
3149 |
Known.One.setAllBits(); |
0 |
| 3150 |
Known.Zero.setAllBits(); |
0 |
3150 |
Known.Zero.setAllBits(); |
0 |
| 3151 |
if (!!DemandedSubElts) { |
0 |
3151 |
if (!!DemandedSubElts) { |
0 |
| 3152 |
Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1); |
0 |
3152 |
Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1); |
0 |
| 3153 |
if (Known.isUnknown()) |
0 |
3153 |
if (Known.isUnknown()) |
0 |
| 3154 |
break; // early-out. |
0 |
3154 |
break; // early-out. |
0 |
| 3155 |
} |
--- |
3155 |
} |
--- |
| 3156 |
if (!!DemandedSrcElts) { |
0 |
3156 |
if (!!DemandedSrcElts) { |
0 |
| 3157 |
Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1); |
0 |
3157 |
Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1); |
0 |
| 3158 |
Known = Known.intersectWith(Known2); |
0 |
3158 |
Known = Known.intersectWith(Known2); |
0 |
| 3159 |
} |
--- |
3159 |
} |
--- |
| 3160 |
break; |
0 |
3160 |
break; |
0 |
| 3161 |
} |
0 |
3161 |
} |
0 |
| 3162 |
case ISD::EXTRACT_SUBVECTOR: { |
0 |
3162 |
case ISD::EXTRACT_SUBVECTOR: { |
0 |
| 3163 |
// Offset the demanded elts by the subvector index. |
--- |
3163 |
// Offset the demanded elts by the subvector index. |
--- |
| 3164 |
SDValue Src = Op.getOperand(0); |
0 |
3164 |
SDValue Src = Op.getOperand(0); |
0 |
| 3165 |
// Bail until we can represent demanded elements for scalable vectors. |
--- |
3165 |
// Bail until we can represent demanded elements for scalable vectors. |
--- |
| 3166 |
if (Op.getValueType().isScalableVector() || Src.getValueType().isScalableVector()) |
0 |
3166 |
if (Op.getValueType().isScalableVector() || Src.getValueType().isScalableVector()) |
0 |
| 3167 |
break; |
0 |
3167 |
break; |
0 |
| 3168 |
uint64_t Idx = Op.getConstantOperandVal(1); |
0 |
3168 |
uint64_t Idx = Op.getConstantOperandVal(1); |
0 |
| 3169 |
unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); |
0 |
3169 |
unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); |
0 |
| 3170 |
APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx); |
0 |
3170 |
APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx); |
0 |
| 3171 |
Known = computeKnownBits(Src, DemandedSrcElts, Depth + 1); |
0 |
3171 |
Known = computeKnownBits(Src, DemandedSrcElts, Depth + 1); |
0 |
| 3172 |
break; |
0 |
3172 |
break; |
0 |
| 3173 |
} |
0 |
3173 |
} |
0 |
| 3174 |
case ISD::SCALAR_TO_VECTOR: { |
0 |
3174 |
case ISD::SCALAR_TO_VECTOR: { |
0 |
| 3175 |
if (Op.getValueType().isScalableVector()) |
0 |
3175 |
if (Op.getValueType().isScalableVector()) |
0 |
| 3176 |
break; |
0 |
3176 |
break; |
0 |
| 3177 |
// We know about scalar_to_vector as much as we know about it source, |
--- |
3177 |
// We know about scalar_to_vector as much as we know about it source, |
--- |
| 3178 |
// which becomes the first element of otherwise unknown vector. |
--- |
3178 |
// which becomes the first element of otherwise unknown vector. |
--- |
| 3179 |
if (DemandedElts != 1) |
0 |
3179 |
if (DemandedElts != 1) |
0 |
| 3180 |
break; |
0 |
3180 |
break; |
0 |
| 3181 |
|
--- |
3181 |
|
--- |
| 3182 |
SDValue N0 = Op.getOperand(0); |
0 |
3182 |
SDValue N0 = Op.getOperand(0); |
0 |
| 3183 |
Known = computeKnownBits(N0, Depth + 1); |
0 |
3183 |
Known = computeKnownBits(N0, Depth + 1); |
0 |
| 3184 |
if (N0.getValueSizeInBits() != BitWidth) |
0 |
3184 |
if (N0.getValueSizeInBits() != BitWidth) |
0 |
| 3185 |
Known = Known.trunc(BitWidth); |
0 |
3185 |
Known = Known.trunc(BitWidth); |
0 |
| 3186 |
|
--- |
3186 |
|
--- |
| 3187 |
break; |
0 |
3187 |
break; |
0 |
| 3188 |
} |
--- |
3188 |
} |
--- |
| 3189 |
case ISD::BITCAST: { |
0 |
3189 |
case ISD::BITCAST: { |
0 |
| 3190 |
if (Op.getValueType().isScalableVector()) |
0 |
3190 |
if (Op.getValueType().isScalableVector()) |
0 |
| 3191 |
break; |
0 |
3191 |
break; |
0 |
| 3192 |
|
--- |
3192 |
|
--- |
| 3193 |
SDValue N0 = Op.getOperand(0); |
0 |
3193 |
SDValue N0 = Op.getOperand(0); |
0 |
| 3194 |
EVT SubVT = N0.getValueType(); |
0 |
3194 |
EVT SubVT = N0.getValueType(); |
0 |
| 3195 |
unsigned SubBitWidth = SubVT.getScalarSizeInBits(); |
0 |
3195 |
unsigned SubBitWidth = SubVT.getScalarSizeInBits(); |
0 |
| 3196 |
|
--- |
3196 |
|
--- |
| 3197 |
// Ignore bitcasts from unsupported types. |
--- |
3197 |
// Ignore bitcasts from unsupported types. |
--- |
| 3198 |
if (!(SubVT.isInteger() || SubVT.isFloatingPoint())) |
0 |
3198 |
if (!(SubVT.isInteger() || SubVT.isFloatingPoint())) |
0 |
| 3199 |
break; |
0 |
3199 |
break; |
0 |
| 3200 |
|
--- |
3200 |
|
--- |
| 3201 |
// Fast handling of 'identity' bitcasts. |
--- |
3201 |
// Fast handling of 'identity' bitcasts. |
--- |
| 3202 |
if (BitWidth == SubBitWidth) { |
0 |
3202 |
if (BitWidth == SubBitWidth) { |
0 |
| 3203 |
Known = computeKnownBits(N0, DemandedElts, Depth + 1); |
0 |
3203 |
Known = computeKnownBits(N0, DemandedElts, Depth + 1); |
0 |
| 3204 |
break; |
0 |
3204 |
break; |
0 |
| 3205 |
} |
--- |
3205 |
} |
--- |
| 3206 |
|
--- |
3206 |
|
--- |
| 3207 |
bool IsLE = getDataLayout().isLittleEndian(); |
0 |
3207 |
bool IsLE = getDataLayout().isLittleEndian(); |
0 |
| 3208 |
|
--- |
3208 |
|
--- |
| 3209 |
// Bitcast 'small element' vector to 'large element' scalar/vector. |
--- |
3209 |
// Bitcast 'small element' vector to 'large element' scalar/vector. |
--- |
| 3210 |
if ((BitWidth % SubBitWidth) == 0) { |
0 |
3210 |
if ((BitWidth % SubBitWidth) == 0) { |
0 |
| 3211 |
assert(N0.getValueType().isVector() && "Expected bitcast from vector"); |
0 |
3211 |
assert(N0.getValueType().isVector() && "Expected bitcast from vector"); |
0 |
| 3212 |
|
--- |
3212 |
|
--- |
| 3213 |
// Collect known bits for the (larger) output by collecting the known |
--- |
3213 |
// Collect known bits for the (larger) output by collecting the known |
--- |
| 3214 |
// bits from each set of sub elements and shift these into place. |
--- |
3214 |
// bits from each set of sub elements and shift these into place. |
--- |
| 3215 |
// We need to separately call computeKnownBits for each set of |
--- |
3215 |
// We need to separately call computeKnownBits for each set of |
--- |
| 3216 |
// sub elements as the knownbits for each is likely to be different. |
--- |
3216 |
// sub elements as the knownbits for each is likely to be different. |
--- |
| 3217 |
unsigned SubScale = BitWidth / SubBitWidth; |
0 |
3217 |
unsigned SubScale = BitWidth / SubBitWidth; |
0 |
| 3218 |
APInt SubDemandedElts(NumElts * SubScale, 0); |
0 |
3218 |
APInt SubDemandedElts(NumElts * SubScale, 0); |
0 |
| 3219 |
for (unsigned i = 0; i != NumElts; ++i) |
0 |
3219 |
for (unsigned i = 0; i != NumElts; ++i) |
0 |
| 3220 |
if (DemandedElts[i]) |
0 |
3220 |
if (DemandedElts[i]) |
0 |
| 3221 |
SubDemandedElts.setBit(i * SubScale); |
0 |
3221 |
SubDemandedElts.setBit(i * SubScale); |
0 |
| 3222 |
|
--- |
3222 |
|
--- |
| 3223 |
for (unsigned i = 0; i != SubScale; ++i) { |
0 |
3223 |
for (unsigned i = 0; i != SubScale; ++i) { |
0 |
| 3224 |
Known2 = computeKnownBits(N0, SubDemandedElts.shl(i), |
0 |
3224 |
Known2 = computeKnownBits(N0, SubDemandedElts.shl(i), |
0 |
| 3225 |
Depth + 1); |
0 |
3225 |
Depth + 1); |
0 |
| 3226 |
unsigned Shifts = IsLE ? i : SubScale - 1 - i; |
0 |
3226 |
unsigned Shifts = IsLE ? i : SubScale - 1 - i; |
0 |
| 3227 |
Known.insertBits(Known2, SubBitWidth * Shifts); |
0 |
3227 |
Known.insertBits(Known2, SubBitWidth * Shifts); |
0 |
| 3228 |
} |
--- |
3228 |
} |
--- |
| 3229 |
} |
0 |
3229 |
} |
0 |
| 3230 |
|
--- |
3230 |
|
--- |
| 3231 |
// Bitcast 'large element' scalar/vector to 'small element' vector. |
--- |
3231 |
// Bitcast 'large element' scalar/vector to 'small element' vector. |
--- |
| 3232 |
if ((SubBitWidth % BitWidth) == 0) { |
0 |
3232 |
if ((SubBitWidth % BitWidth) == 0) { |
0 |
| 3233 |
assert(Op.getValueType().isVector() && "Expected bitcast to vector"); |
0 |
3233 |
assert(Op.getValueType().isVector() && "Expected bitcast to vector"); |
0 |
| 3234 |
|
--- |
3234 |
|
--- |
| 3235 |
// Collect known bits for the (smaller) output by collecting the known |
--- |
3235 |
// Collect known bits for the (smaller) output by collecting the known |
--- |
| 3236 |
// bits from the overlapping larger input elements and extracting the |
--- |
3236 |
// bits from the overlapping larger input elements and extracting the |
--- |
| 3237 |
// sub sections we actually care about. |
--- |
3237 |
// sub sections we actually care about. |
--- |
| 3238 |
unsigned SubScale = SubBitWidth / BitWidth; |
0 |
3238 |
unsigned SubScale = SubBitWidth / BitWidth; |
0 |
| 3239 |
APInt SubDemandedElts = |
--- |
3239 |
APInt SubDemandedElts = |
--- |
| 3240 |
APIntOps::ScaleBitMask(DemandedElts, NumElts / SubScale); |
0 |
3240 |
APIntOps::ScaleBitMask(DemandedElts, NumElts / SubScale); |
0 |
| 3241 |
Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1); |
0 |
3241 |
Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1); |
0 |
| 3242 |
|
--- |
3242 |
|
--- |
| 3243 |
Known.Zero.setAllBits(); Known.One.setAllBits(); |
0 |
3243 |
Known.Zero.setAllBits(); Known.One.setAllBits(); |
0 |
| 3244 |
for (unsigned i = 0; i != NumElts; ++i) |
0 |
3244 |
for (unsigned i = 0; i != NumElts; ++i) |
0 |
| 3245 |
if (DemandedElts[i]) { |
0 |
3245 |
if (DemandedElts[i]) { |
0 |
| 3246 |
unsigned Shifts = IsLE ? i : NumElts - 1 - i; |
0 |
3246 |
unsigned Shifts = IsLE ? i : NumElts - 1 - i; |
0 |
| 3247 |
unsigned Offset = (Shifts % SubScale) * BitWidth; |
0 |
3247 |
unsigned Offset = (Shifts % SubScale) * BitWidth; |
0 |
| 3248 |
Known = Known.intersectWith(Known2.extractBits(BitWidth, Offset)); |
0 |
3248 |
Known = Known.intersectWith(Known2.extractBits(BitWidth, Offset)); |
0 |
| 3249 |
// If we don't know any bits, early out. |
--- |
3249 |
// If we don't know any bits, early out. |
--- |
| 3250 |
if (Known.isUnknown()) |
0 |
3250 |
if (Known.isUnknown()) |
0 |
| 3251 |
break; |
0 |
3251 |
break; |
0 |
| 3252 |
} |
--- |
3252 |
} |
--- |
| 3253 |
} |
0 |
3253 |
} |
0 |
| 3254 |
break; |
0 |
3254 |
break; |
0 |
| 3255 |
} |
--- |
3255 |
} |
--- |
| 3256 |
case ISD::AND: |
1 |
3256 |
case ISD::AND: |
1 |
| 3257 |
Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
1 |
3257 |
Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
1 |
| 3258 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
1 |
3258 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
1 |
| 3259 |
|
--- |
3259 |
|
--- |
| 3260 |
Known &= Known2; |
1 |
3260 |
Known &= Known2; |
1 |
| 3261 |
break; |
1 |
3261 |
break; |
1 |
| 3262 |
case ISD::OR: |
0 |
3262 |
case ISD::OR: |
0 |
| 3263 |
Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3263 |
Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3264 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3264 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3265 |
|
--- |
3265 |
|
--- |
| 3266 |
Known |= Known2; |
0 |
3266 |
Known |= Known2; |
0 |
| 3267 |
break; |
0 |
3267 |
break; |
0 |
| 3268 |
case ISD::XOR: |
0 |
3268 |
case ISD::XOR: |
0 |
| 3269 |
Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3269 |
Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3270 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3270 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3271 |
|
--- |
3271 |
|
--- |
| 3272 |
Known ^= Known2; |
0 |
3272 |
Known ^= Known2; |
0 |
| 3273 |
break; |
0 |
3273 |
break; |
0 |
| 3274 |
case ISD::MUL: { |
0 |
3274 |
case ISD::MUL: { |
0 |
| 3275 |
Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3275 |
Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3276 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3276 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3277 |
bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1); |
0 |
3277 |
bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1); |
0 |
| 3278 |
// TODO: SelfMultiply can be poison, but not undef. |
--- |
3278 |
// TODO: SelfMultiply can be poison, but not undef. |
--- |
| 3279 |
if (SelfMultiply) |
0 |
3279 |
if (SelfMultiply) |
0 |
| 3280 |
SelfMultiply &= isGuaranteedNotToBeUndefOrPoison( |
0 |
3280 |
SelfMultiply &= isGuaranteedNotToBeUndefOrPoison( |
0 |
| 3281 |
Op.getOperand(0), DemandedElts, false, Depth + 1); |
0 |
3281 |
Op.getOperand(0), DemandedElts, false, Depth + 1); |
0 |
| 3282 |
Known = KnownBits::mul(Known, Known2, SelfMultiply); |
0 |
3282 |
Known = KnownBits::mul(Known, Known2, SelfMultiply); |
0 |
| 3283 |
|
--- |
3283 |
|
--- |
| 3284 |
// If the multiplication is known not to overflow, the product of a number |
--- |
3284 |
// If the multiplication is known not to overflow, the product of a number |
--- |
| 3285 |
// with itself is non-negative. Only do this if we didn't already computed |
--- |
3285 |
// with itself is non-negative. Only do this if we didn't already computed |
--- |
| 3286 |
// the opposite value for the sign bit. |
--- |
3286 |
// the opposite value for the sign bit. |
--- |
| 3287 |
if (Op->getFlags().hasNoSignedWrap() && |
0 |
3287 |
if (Op->getFlags().hasNoSignedWrap() && |
0 |
| 3288 |
Op.getOperand(0) == Op.getOperand(1) && |
0 |
3288 |
Op.getOperand(0) == Op.getOperand(1) && |
0 |
| 3289 |
!Known.isNegative()) |
0 |
3289 |
!Known.isNegative()) |
0 |
| 3290 |
Known.makeNonNegative(); |
0 |
3290 |
Known.makeNonNegative(); |
0 |
| 3291 |
break; |
0 |
3291 |
break; |
0 |
| 3292 |
} |
--- |
3292 |
} |
--- |
| 3293 |
case ISD::MULHU: { |
0 |
3293 |
case ISD::MULHU: { |
0 |
| 3294 |
Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3294 |
Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3295 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3295 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3296 |
Known = KnownBits::mulhu(Known, Known2); |
0 |
3296 |
Known = KnownBits::mulhu(Known, Known2); |
0 |
| 3297 |
break; |
0 |
3297 |
break; |
0 |
| 3298 |
} |
--- |
3298 |
} |
--- |
| 3299 |
case ISD::MULHS: { |
0 |
3299 |
case ISD::MULHS: { |
0 |
| 3300 |
Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3300 |
Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3301 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3301 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3302 |
Known = KnownBits::mulhs(Known, Known2); |
0 |
3302 |
Known = KnownBits::mulhs(Known, Known2); |
0 |
| 3303 |
break; |
0 |
3303 |
break; |
0 |
| 3304 |
} |
--- |
3304 |
} |
--- |
| 3305 |
case ISD::UMUL_LOHI: { |
0 |
3305 |
case ISD::UMUL_LOHI: { |
0 |
| 3306 |
assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result"); |
0 |
3306 |
assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result"); |
0 |
| 3307 |
Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3307 |
Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3308 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3308 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3309 |
bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1); |
0 |
3309 |
bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1); |
0 |
| 3310 |
if (Op.getResNo() == 0) |
0 |
3310 |
if (Op.getResNo() == 0) |
0 |
| 3311 |
Known = KnownBits::mul(Known, Known2, SelfMultiply); |
0 |
3311 |
Known = KnownBits::mul(Known, Known2, SelfMultiply); |
0 |
| 3312 |
else |
--- |
3312 |
else |
--- |
| 3313 |
Known = KnownBits::mulhu(Known, Known2); |
0 |
3313 |
Known = KnownBits::mulhu(Known, Known2); |
0 |
| 3314 |
break; |
0 |
3314 |
break; |
0 |
| 3315 |
} |
--- |
3315 |
} |
--- |
| 3316 |
case ISD::SMUL_LOHI: { |
0 |
3316 |
case ISD::SMUL_LOHI: { |
0 |
| 3317 |
assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result"); |
0 |
3317 |
assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result"); |
0 |
| 3318 |
Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3318 |
Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3319 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3319 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3320 |
bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1); |
0 |
3320 |
bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1); |
0 |
| 3321 |
if (Op.getResNo() == 0) |
0 |
3321 |
if (Op.getResNo() == 0) |
0 |
| 3322 |
Known = KnownBits::mul(Known, Known2, SelfMultiply); |
0 |
3322 |
Known = KnownBits::mul(Known, Known2, SelfMultiply); |
0 |
| 3323 |
else |
--- |
3323 |
else |
--- |
| 3324 |
Known = KnownBits::mulhs(Known, Known2); |
0 |
3324 |
Known = KnownBits::mulhs(Known, Known2); |
0 |
| 3325 |
break; |
0 |
3325 |
break; |
0 |
| 3326 |
} |
--- |
3326 |
} |
--- |
| 3327 |
case ISD::AVGCEILU: { |
0 |
3327 |
case ISD::AVGCEILU: { |
0 |
| 3328 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3328 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3329 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3329 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3330 |
Known = Known.zext(BitWidth + 1); |
0 |
3330 |
Known = Known.zext(BitWidth + 1); |
0 |
| 3331 |
Known2 = Known2.zext(BitWidth + 1); |
0 |
3331 |
Known2 = Known2.zext(BitWidth + 1); |
0 |
| 3332 |
KnownBits One = KnownBits::makeConstant(APInt(1, 1)); |
0 |
3332 |
KnownBits One = KnownBits::makeConstant(APInt(1, 1)); |
0 |
| 3333 |
Known = KnownBits::computeForAddCarry(Known, Known2, One); |
0 |
3333 |
Known = KnownBits::computeForAddCarry(Known, Known2, One); |
0 |
| 3334 |
Known = Known.extractBits(BitWidth, 1); |
0 |
3334 |
Known = Known.extractBits(BitWidth, 1); |
0 |
| 3335 |
break; |
0 |
3335 |
break; |
0 |
| 3336 |
} |
0 |
3336 |
} |
0 |
| 3337 |
case ISD::SELECT: |
0 |
3337 |
case ISD::SELECT: |
0 |
| 3338 |
case ISD::VSELECT: |
--- |
3338 |
case ISD::VSELECT: |
--- |
| 3339 |
Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1); |
0 |
3339 |
Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1); |
0 |
| 3340 |
// If we don't know any bits, early out. |
--- |
3340 |
// If we don't know any bits, early out. |
--- |
| 3341 |
if (Known.isUnknown()) |
0 |
3341 |
if (Known.isUnknown()) |
0 |
| 3342 |
break; |
0 |
3342 |
break; |
0 |
| 3343 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1); |
0 |
3343 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1); |
0 |
| 3344 |
|
--- |
3344 |
|
--- |
| 3345 |
// Only known if known in both the LHS and RHS. |
--- |
3345 |
// Only known if known in both the LHS and RHS. |
--- |
| 3346 |
Known = Known.intersectWith(Known2); |
0 |
3346 |
Known = Known.intersectWith(Known2); |
0 |
| 3347 |
break; |
0 |
3347 |
break; |
0 |
| 3348 |
case ISD::SELECT_CC: |
0 |
3348 |
case ISD::SELECT_CC: |
0 |
| 3349 |
Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1); |
0 |
3349 |
Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1); |
0 |
| 3350 |
// If we don't know any bits, early out. |
--- |
3350 |
// If we don't know any bits, early out. |
--- |
| 3351 |
if (Known.isUnknown()) |
0 |
3351 |
if (Known.isUnknown()) |
0 |
| 3352 |
break; |
0 |
3352 |
break; |
0 |
| 3353 |
Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1); |
0 |
3353 |
Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1); |
0 |
| 3354 |
|
--- |
3354 |
|
--- |
| 3355 |
// Only known if known in both the LHS and RHS. |
--- |
3355 |
// Only known if known in both the LHS and RHS. |
--- |
| 3356 |
Known = Known.intersectWith(Known2); |
0 |
3356 |
Known = Known.intersectWith(Known2); |
0 |
| 3357 |
break; |
0 |
3357 |
break; |
0 |
| 3358 |
case ISD::SMULO: |
0 |
3358 |
case ISD::SMULO: |
0 |
| 3359 |
case ISD::UMULO: |
--- |
3359 |
case ISD::UMULO: |
--- |
| 3360 |
if (Op.getResNo() != 1) |
0 |
3360 |
if (Op.getResNo() != 1) |
0 |
| 3361 |
break; |
0 |
3361 |
break; |
0 |
| 3362 |
// The boolean result conforms to getBooleanContents. |
--- |
3362 |
// The boolean result conforms to getBooleanContents. |
--- |
| 3363 |
// If we know the result of a setcc has the top bits zero, use this info. |
--- |
3363 |
// If we know the result of a setcc has the top bits zero, use this info. |
--- |
| 3364 |
// We know that we have an integer-based boolean since these operations |
--- |
3364 |
// We know that we have an integer-based boolean since these operations |
--- |
| 3365 |
// are only available for integer. |
--- |
3365 |
// are only available for integer. |
--- |
| 3366 |
if (TLI->getBooleanContents(Op.getValueType().isVector(), false) == |
0 |
3366 |
if (TLI->getBooleanContents(Op.getValueType().isVector(), false) == |
0 |
| 3367 |
TargetLowering::ZeroOrOneBooleanContent && |
0 |
3367 |
TargetLowering::ZeroOrOneBooleanContent && |
0 |
| 3368 |
BitWidth > 1) |
--- |
3368 |
BitWidth > 1) |
--- |
| 3369 |
Known.Zero.setBitsFrom(1); |
0 |
3369 |
Known.Zero.setBitsFrom(1); |
0 |
| 3370 |
break; |
0 |
3370 |
break; |
0 |
| 3371 |
case ISD::SETCC: |
3 |
3371 |
case ISD::SETCC: |
3 |
| 3372 |
case ISD::SETCCCARRY: |
--- |
3372 |
case ISD::SETCCCARRY: |
--- |
| 3373 |
case ISD::STRICT_FSETCC: |
--- |
3373 |
case ISD::STRICT_FSETCC: |
--- |
| 3374 |
case ISD::STRICT_FSETCCS: { |
--- |
3374 |
case ISD::STRICT_FSETCCS: { |
--- |
| 3375 |
unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0; |
3 |
3375 |
unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0; |
3 |
| 3376 |
// If we know the result of a setcc has the top bits zero, use this info. |
--- |
3376 |
// If we know the result of a setcc has the top bits zero, use this info. |
--- |
| 3377 |
if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) == |
3 |
3377 |
if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) == |
3 |
| 3378 |
TargetLowering::ZeroOrOneBooleanContent && |
3 |
3378 |
TargetLowering::ZeroOrOneBooleanContent && |
3 |
| 3379 |
BitWidth > 1) |
--- |
3379 |
BitWidth > 1) |
--- |
| 3380 |
Known.Zero.setBitsFrom(1); |
2 |
3380 |
Known.Zero.setBitsFrom(1); |
2 |
| 3381 |
break; |
3 |
3381 |
break; |
3 |
| 3382 |
} |
--- |
3382 |
} |
--- |
| 3383 |
case ISD::SHL: |
0 |
3383 |
case ISD::SHL: |
0 |
| 3384 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3384 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3385 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3385 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3386 |
Known = KnownBits::shl(Known, Known2); |
0 |
3386 |
Known = KnownBits::shl(Known, Known2); |
0 |
| 3387 |
|
--- |
3387 |
|
--- |
| 3388 |
// Minimum shift low bits are known zero. |
--- |
3388 |
// Minimum shift low bits are known zero. |
--- |
| 3389 |
if (const APInt *ShMinAmt = |
0 |
3389 |
if (const APInt *ShMinAmt = |
0 |
| 3390 |
getValidMinimumShiftAmountConstant(Op, DemandedElts)) |
0 |
3390 |
getValidMinimumShiftAmountConstant(Op, DemandedElts)) |
0 |
| 3391 |
Known.Zero.setLowBits(ShMinAmt->getZExtValue()); |
0 |
3391 |
Known.Zero.setLowBits(ShMinAmt->getZExtValue()); |
0 |
| 3392 |
break; |
0 |
3392 |
break; |
0 |
| 3393 |
case ISD::SRL: |
0 |
3393 |
case ISD::SRL: |
0 |
| 3394 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3394 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3395 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3395 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3396 |
Known = KnownBits::lshr(Known, Known2); |
0 |
3396 |
Known = KnownBits::lshr(Known, Known2); |
0 |
| 3397 |
|
--- |
3397 |
|
--- |
| 3398 |
// Minimum shift high bits are known zero. |
--- |
3398 |
// Minimum shift high bits are known zero. |
--- |
| 3399 |
if (const APInt *ShMinAmt = |
0 |
3399 |
if (const APInt *ShMinAmt = |
0 |
| 3400 |
getValidMinimumShiftAmountConstant(Op, DemandedElts)) |
0 |
3400 |
getValidMinimumShiftAmountConstant(Op, DemandedElts)) |
0 |
| 3401 |
Known.Zero.setHighBits(ShMinAmt->getZExtValue()); |
0 |
3401 |
Known.Zero.setHighBits(ShMinAmt->getZExtValue()); |
0 |
| 3402 |
break; |
0 |
3402 |
break; |
0 |
| 3403 |
case ISD::SRA: |
0 |
3403 |
case ISD::SRA: |
0 |
| 3404 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3404 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3405 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3405 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3406 |
Known = KnownBits::ashr(Known, Known2); |
0 |
3406 |
Known = KnownBits::ashr(Known, Known2); |
0 |
| 3407 |
break; |
0 |
3407 |
break; |
0 |
| 3408 |
case ISD::FSHL: |
0 |
3408 |
case ISD::FSHL: |
0 |
| 3409 |
case ISD::FSHR: |
--- |
3409 |
case ISD::FSHR: |
--- |
| 3410 |
if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) { |
0 |
3410 |
if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) { |
0 |
| 3411 |
unsigned Amt = C->getAPIntValue().urem(BitWidth); |
0 |
3411 |
unsigned Amt = C->getAPIntValue().urem(BitWidth); |
0 |
| 3412 |
|
--- |
3412 |
|
--- |
| 3413 |
// For fshl, 0-shift returns the 1st arg. |
--- |
3413 |
// For fshl, 0-shift returns the 1st arg. |
--- |
| 3414 |
// For fshr, 0-shift returns the 2nd arg. |
--- |
3414 |
// For fshr, 0-shift returns the 2nd arg. |
--- |
| 3415 |
if (Amt == 0) { |
0 |
3415 |
if (Amt == 0) { |
0 |
| 3416 |
Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1), |
0 |
3416 |
Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1), |
0 |
| 3417 |
DemandedElts, Depth + 1); |
0 |
3417 |
DemandedElts, Depth + 1); |
0 |
| 3418 |
break; |
0 |
3418 |
break; |
0 |
| 3419 |
} |
--- |
3419 |
} |
--- |
| 3420 |
|
--- |
3420 |
|
--- |
| 3421 |
// fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW))) |
--- |
3421 |
// fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW))) |
--- |
| 3422 |
// fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW)) |
--- |
3422 |
// fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW)) |
--- |
| 3423 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3423 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3424 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3424 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3425 |
if (Opcode == ISD::FSHL) { |
0 |
3425 |
if (Opcode == ISD::FSHL) { |
0 |
| 3426 |
Known.One <<= Amt; |
0 |
3426 |
Known.One <<= Amt; |
0 |
| 3427 |
Known.Zero <<= Amt; |
0 |
3427 |
Known.Zero <<= Amt; |
0 |
| 3428 |
Known2.One.lshrInPlace(BitWidth - Amt); |
0 |
3428 |
Known2.One.lshrInPlace(BitWidth - Amt); |
0 |
| 3429 |
Known2.Zero.lshrInPlace(BitWidth - Amt); |
0 |
3429 |
Known2.Zero.lshrInPlace(BitWidth - Amt); |
0 |
| 3430 |
} else { |
--- |
3430 |
} else { |
--- |
| 3431 |
Known.One <<= BitWidth - Amt; |
0 |
3431 |
Known.One <<= BitWidth - Amt; |
0 |
| 3432 |
Known.Zero <<= BitWidth - Amt; |
0 |
3432 |
Known.Zero <<= BitWidth - Amt; |
0 |
| 3433 |
Known2.One.lshrInPlace(Amt); |
0 |
3433 |
Known2.One.lshrInPlace(Amt); |
0 |
| 3434 |
Known2.Zero.lshrInPlace(Amt); |
0 |
3434 |
Known2.Zero.lshrInPlace(Amt); |
0 |
| 3435 |
} |
--- |
3435 |
} |
--- |
| 3436 |
Known = Known.unionWith(Known2); |
0 |
3436 |
Known = Known.unionWith(Known2); |
0 |
| 3437 |
} |
--- |
3437 |
} |
--- |
| 3438 |
break; |
0 |
3438 |
break; |
0 |
| 3439 |
case ISD::SHL_PARTS: |
0 |
3439 |
case ISD::SHL_PARTS: |
0 |
| 3440 |
case ISD::SRA_PARTS: |
--- |
3440 |
case ISD::SRA_PARTS: |
--- |
| 3441 |
case ISD::SRL_PARTS: { |
--- |
3441 |
case ISD::SRL_PARTS: { |
--- |
| 3442 |
assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result"); |
0 |
3442 |
assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result"); |
0 |
| 3443 |
|
--- |
3443 |
|
--- |
| 3444 |
// Collect lo/hi source values and concatenate. |
--- |
3444 |
// Collect lo/hi source values and concatenate. |
--- |
| 3445 |
unsigned LoBits = Op.getOperand(0).getScalarValueSizeInBits(); |
0 |
3445 |
unsigned LoBits = Op.getOperand(0).getScalarValueSizeInBits(); |
0 |
| 3446 |
unsigned HiBits = Op.getOperand(1).getScalarValueSizeInBits(); |
0 |
3446 |
unsigned HiBits = Op.getOperand(1).getScalarValueSizeInBits(); |
0 |
| 3447 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3447 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3448 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3448 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3449 |
Known = Known2.concat(Known); |
0 |
3449 |
Known = Known2.concat(Known); |
0 |
| 3450 |
|
--- |
3450 |
|
--- |
| 3451 |
// Collect shift amount. |
--- |
3451 |
// Collect shift amount. |
--- |
| 3452 |
Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1); |
0 |
3452 |
Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1); |
0 |
| 3453 |
|
--- |
3453 |
|
--- |
| 3454 |
if (Opcode == ISD::SHL_PARTS) |
0 |
3454 |
if (Opcode == ISD::SHL_PARTS) |
0 |
| 3455 |
Known = KnownBits::shl(Known, Known2); |
0 |
3455 |
Known = KnownBits::shl(Known, Known2); |
0 |
| 3456 |
else if (Opcode == ISD::SRA_PARTS) |
0 |
3456 |
else if (Opcode == ISD::SRA_PARTS) |
0 |
| 3457 |
Known = KnownBits::ashr(Known, Known2); |
0 |
3457 |
Known = KnownBits::ashr(Known, Known2); |
0 |
| 3458 |
else // if (Opcode == ISD::SRL_PARTS) |
--- |
3458 |
else // if (Opcode == ISD::SRL_PARTS) |
--- |
| 3459 |
Known = KnownBits::lshr(Known, Known2); |
0 |
3459 |
Known = KnownBits::lshr(Known, Known2); |
0 |
| 3460 |
|
--- |
3460 |
|
--- |
| 3461 |
// TODO: Minimum shift low/high bits are known zero. |
--- |
3461 |
// TODO: Minimum shift low/high bits are known zero. |
--- |
| 3462 |
|
--- |
3462 |
|
--- |
| 3463 |
if (Op.getResNo() == 0) |
0 |
3463 |
if (Op.getResNo() == 0) |
0 |
| 3464 |
Known = Known.extractBits(LoBits, 0); |
0 |
3464 |
Known = Known.extractBits(LoBits, 0); |
0 |
| 3465 |
else |
--- |
3465 |
else |
--- |
| 3466 |
Known = Known.extractBits(HiBits, LoBits); |
0 |
3466 |
Known = Known.extractBits(HiBits, LoBits); |
0 |
| 3467 |
break; |
0 |
3467 |
break; |
0 |
| 3468 |
} |
--- |
3468 |
} |
--- |
| 3469 |
case ISD::SIGN_EXTEND_INREG: { |
0 |
3469 |
case ISD::SIGN_EXTEND_INREG: { |
0 |
| 3470 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3470 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3471 |
EVT EVT = cast(Op.getOperand(1))->getVT(); |
0 |
3471 |
EVT EVT = cast(Op.getOperand(1))->getVT(); |
0 |
| 3472 |
Known = Known.sextInReg(EVT.getScalarSizeInBits()); |
0 |
3472 |
Known = Known.sextInReg(EVT.getScalarSizeInBits()); |
0 |
| 3473 |
break; |
0 |
3473 |
break; |
0 |
| 3474 |
} |
--- |
3474 |
} |
--- |
| 3475 |
case ISD::CTTZ: |
0 |
3475 |
case ISD::CTTZ: |
0 |
| 3476 |
case ISD::CTTZ_ZERO_UNDEF: { |
--- |
3476 |
case ISD::CTTZ_ZERO_UNDEF: { |
--- |
| 3477 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3477 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3478 |
// If we have a known 1, its position is our upper bound. |
--- |
3478 |
// If we have a known 1, its position is our upper bound. |
--- |
| 3479 |
unsigned PossibleTZ = Known2.countMaxTrailingZeros(); |
0 |
3479 |
unsigned PossibleTZ = Known2.countMaxTrailingZeros(); |
0 |
| 3480 |
unsigned LowBits = llvm::bit_width(PossibleTZ); |
0 |
3480 |
unsigned LowBits = llvm::bit_width(PossibleTZ); |
0 |
| 3481 |
Known.Zero.setBitsFrom(LowBits); |
0 |
3481 |
Known.Zero.setBitsFrom(LowBits); |
0 |
| 3482 |
break; |
0 |
3482 |
break; |
0 |
| 3483 |
} |
--- |
3483 |
} |
--- |
| 3484 |
case ISD::CTLZ: |
0 |
3484 |
case ISD::CTLZ: |
0 |
| 3485 |
case ISD::CTLZ_ZERO_UNDEF: { |
--- |
3485 |
case ISD::CTLZ_ZERO_UNDEF: { |
--- |
| 3486 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3486 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3487 |
// If we have a known 1, its position is our upper bound. |
--- |
3487 |
// If we have a known 1, its position is our upper bound. |
--- |
| 3488 |
unsigned PossibleLZ = Known2.countMaxLeadingZeros(); |
0 |
3488 |
unsigned PossibleLZ = Known2.countMaxLeadingZeros(); |
0 |
| 3489 |
unsigned LowBits = llvm::bit_width(PossibleLZ); |
0 |
3489 |
unsigned LowBits = llvm::bit_width(PossibleLZ); |
0 |
| 3490 |
Known.Zero.setBitsFrom(LowBits); |
0 |
3490 |
Known.Zero.setBitsFrom(LowBits); |
0 |
| 3491 |
break; |
0 |
3491 |
break; |
0 |
| 3492 |
} |
--- |
3492 |
} |
--- |
| 3493 |
case ISD::CTPOP: { |
0 |
3493 |
case ISD::CTPOP: { |
0 |
| 3494 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3494 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3495 |
// If we know some of the bits are zero, they can't be one. |
--- |
3495 |
// If we know some of the bits are zero, they can't be one. |
--- |
| 3496 |
unsigned PossibleOnes = Known2.countMaxPopulation(); |
0 |
3496 |
unsigned PossibleOnes = Known2.countMaxPopulation(); |
0 |
| 3497 |
Known.Zero.setBitsFrom(llvm::bit_width(PossibleOnes)); |
0 |
3497 |
Known.Zero.setBitsFrom(llvm::bit_width(PossibleOnes)); |
0 |
| 3498 |
break; |
0 |
3498 |
break; |
0 |
| 3499 |
} |
--- |
3499 |
} |
--- |
| 3500 |
case ISD::PARITY: { |
0 |
3500 |
case ISD::PARITY: { |
0 |
| 3501 |
// Parity returns 0 everywhere but the LSB. |
--- |
3501 |
// Parity returns 0 everywhere but the LSB. |
--- |
| 3502 |
Known.Zero.setBitsFrom(1); |
0 |
3502 |
Known.Zero.setBitsFrom(1); |
0 |
| 3503 |
break; |
0 |
3503 |
break; |
0 |
| 3504 |
} |
--- |
3504 |
} |
--- |
| 3505 |
case ISD::LOAD: { |
0 |
3505 |
case ISD::LOAD: { |
0 |
| 3506 |
LoadSDNode *LD = cast(Op); |
0 |
3506 |
LoadSDNode *LD = cast(Op); |
0 |
| 3507 |
const Constant *Cst = TLI->getTargetConstantFromLoad(LD); |
0 |
3507 |
const Constant *Cst = TLI->getTargetConstantFromLoad(LD); |
0 |
| 3508 |
if (ISD::isNON_EXTLoad(LD) && Cst) { |
0 |
3508 |
if (ISD::isNON_EXTLoad(LD) && Cst) { |
0 |
| 3509 |
// Determine any common known bits from the loaded constant pool value. |
--- |
3509 |
// Determine any common known bits from the loaded constant pool value. |
--- |
| 3510 |
Type *CstTy = Cst->getType(); |
0 |
3510 |
Type *CstTy = Cst->getType(); |
0 |
| 3511 |
if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() && |
0 |
3511 |
if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() && |
0 |
| 3512 |
!Op.getValueType().isScalableVector()) { |
0 |
3512 |
!Op.getValueType().isScalableVector()) { |
0 |
| 3513 |
// If its a vector splat, then we can (quickly) reuse the scalar path. |
--- |
3513 |
// If its a vector splat, then we can (quickly) reuse the scalar path. |
--- |
| 3514 |
// NOTE: We assume all elements match and none are UNDEF. |
--- |
3514 |
// NOTE: We assume all elements match and none are UNDEF. |
--- |
| 3515 |
if (CstTy->isVectorTy()) { |
0 |
3515 |
if (CstTy->isVectorTy()) { |
0 |
| 3516 |
if (const Constant *Splat = Cst->getSplatValue()) { |
0 |
3516 |
if (const Constant *Splat = Cst->getSplatValue()) { |
0 |
| 3517 |
Cst = Splat; |
0 |
3517 |
Cst = Splat; |
0 |
| 3518 |
CstTy = Cst->getType(); |
0 |
3518 |
CstTy = Cst->getType(); |
0 |
| 3519 |
} |
--- |
3519 |
} |
--- |
| 3520 |
} |
--- |
3520 |
} |
--- |
| 3521 |
// TODO - do we need to handle different bitwidths? |
--- |
3521 |
// TODO - do we need to handle different bitwidths? |
--- |
| 3522 |
if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) { |
0 |
3522 |
if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) { |
0 |
| 3523 |
// Iterate across all vector elements finding common known bits. |
--- |
3523 |
// Iterate across all vector elements finding common known bits. |
--- |
| 3524 |
Known.One.setAllBits(); |
0 |
3524 |
Known.One.setAllBits(); |
0 |
| 3525 |
Known.Zero.setAllBits(); |
0 |
3525 |
Known.Zero.setAllBits(); |
0 |
| 3526 |
for (unsigned i = 0; i != NumElts; ++i) { |
0 |
3526 |
for (unsigned i = 0; i != NumElts; ++i) { |
0 |
| 3527 |
if (!DemandedElts[i]) |
0 |
3527 |
if (!DemandedElts[i]) |
0 |
| 3528 |
continue; |
0 |
3528 |
continue; |
0 |
| 3529 |
if (Constant *Elt = Cst->getAggregateElement(i)) { |
0 |
3529 |
if (Constant *Elt = Cst->getAggregateElement(i)) { |
0 |
| 3530 |
if (auto *CInt = dyn_cast(Elt)) { |
0 |
3530 |
if (auto *CInt = dyn_cast(Elt)) { |
0 |
| 3531 |
const APInt &Value = CInt->getValue(); |
0 |
3531 |
const APInt &Value = CInt->getValue(); |
0 |
| 3532 |
Known.One &= Value; |
0 |
3532 |
Known.One &= Value; |
0 |
| 3533 |
Known.Zero &= ~Value; |
0 |
3533 |
Known.Zero &= ~Value; |
0 |
| 3534 |
continue; |
0 |
3534 |
continue; |
0 |
| 3535 |
} |
0 |
3535 |
} |
0 |
| 3536 |
if (auto *CFP = dyn_cast(Elt)) { |
0 |
3536 |
if (auto *CFP = dyn_cast(Elt)) { |
0 |
| 3537 |
APInt Value = CFP->getValueAPF().bitcastToAPInt(); |
0 |
3537 |
APInt Value = CFP->getValueAPF().bitcastToAPInt(); |
0 |
| 3538 |
Known.One &= Value; |
0 |
3538 |
Known.One &= Value; |
0 |
| 3539 |
Known.Zero &= ~Value; |
0 |
3539 |
Known.Zero &= ~Value; |
0 |
| 3540 |
continue; |
0 |
3540 |
continue; |
0 |
| 3541 |
} |
0 |
3541 |
} |
0 |
| 3542 |
} |
--- |
3542 |
} |
--- |
| 3543 |
Known.One.clearAllBits(); |
0 |
3543 |
Known.One.clearAllBits(); |
0 |
| 3544 |
Known.Zero.clearAllBits(); |
0 |
3544 |
Known.Zero.clearAllBits(); |
0 |
| 3545 |
break; |
0 |
3545 |
break; |
0 |
| 3546 |
} |
--- |
3546 |
} |
--- |
| 3547 |
} else if (BitWidth == CstTy->getPrimitiveSizeInBits()) { |
0 |
3547 |
} else if (BitWidth == CstTy->getPrimitiveSizeInBits()) { |
0 |
| 3548 |
if (auto *CInt = dyn_cast(Cst)) { |
0 |
3548 |
if (auto *CInt = dyn_cast(Cst)) { |
0 |
| 3549 |
Known = KnownBits::makeConstant(CInt->getValue()); |
0 |
3549 |
Known = KnownBits::makeConstant(CInt->getValue()); |
0 |
| 3550 |
} else if (auto *CFP = dyn_cast(Cst)) { |
0 |
3550 |
} else if (auto *CFP = dyn_cast(Cst)) { |
0 |
| 3551 |
Known = |
--- |
3551 |
Known = |
--- |
| 3552 |
KnownBits::makeConstant(CFP->getValueAPF().bitcastToAPInt()); |
0 |
3552 |
KnownBits::makeConstant(CFP->getValueAPF().bitcastToAPInt()); |
0 |
| 3553 |
} |
--- |
3553 |
} |
--- |
| 3554 |
} |
--- |
3554 |
} |
--- |
| 3555 |
} |
--- |
3555 |
} |
--- |
| 3556 |
} else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { |
0 |
3556 |
} else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { |
0 |
| 3557 |
// If this is a ZEXTLoad and we are looking at the loaded value. |
--- |
3557 |
// If this is a ZEXTLoad and we are looking at the loaded value. |
--- |
| 3558 |
EVT VT = LD->getMemoryVT(); |
0 |
3558 |
EVT VT = LD->getMemoryVT(); |
0 |
| 3559 |
unsigned MemBits = VT.getScalarSizeInBits(); |
0 |
3559 |
unsigned MemBits = VT.getScalarSizeInBits(); |
0 |
| 3560 |
Known.Zero.setBitsFrom(MemBits); |
0 |
3560 |
Known.Zero.setBitsFrom(MemBits); |
0 |
| 3561 |
} else if (const MDNode *Ranges = LD->getRanges()) { |
0 |
3561 |
} else if (const MDNode *Ranges = LD->getRanges()) { |
0 |
| 3562 |
EVT VT = LD->getValueType(0); |
0 |
3562 |
EVT VT = LD->getValueType(0); |
0 |
| 3563 |
|
--- |
3563 |
|
--- |
| 3564 |
// TODO: Handle for extending loads |
--- |
3564 |
// TODO: Handle for extending loads |
--- |
| 3565 |
if (LD->getExtensionType() == ISD::NON_EXTLOAD) { |
0 |
3565 |
if (LD->getExtensionType() == ISD::NON_EXTLOAD) { |
0 |
| 3566 |
if (VT.isVector()) { |
0 |
3566 |
if (VT.isVector()) { |
0 |
| 3567 |
// Handle truncation to the first demanded element. |
--- |
3567 |
// Handle truncation to the first demanded element. |
--- |
| 3568 |
// TODO: Figure out which demanded elements are covered |
--- |
3568 |
// TODO: Figure out which demanded elements are covered |
--- |
| 3569 |
if (DemandedElts != 1 || !getDataLayout().isLittleEndian()) |
0 |
3569 |
if (DemandedElts != 1 || !getDataLayout().isLittleEndian()) |
0 |
| 3570 |
break; |
0 |
3570 |
break; |
0 |
| 3571 |
|
--- |
3571 |
|
--- |
| 3572 |
// Handle the case where a load has a vector type, but scalar memory |
--- |
3572 |
// Handle the case where a load has a vector type, but scalar memory |
--- |
| 3573 |
// with an attached range. |
--- |
3573 |
// with an attached range. |
--- |
| 3574 |
EVT MemVT = LD->getMemoryVT(); |
0 |
3574 |
EVT MemVT = LD->getMemoryVT(); |
0 |
| 3575 |
KnownBits KnownFull(MemVT.getSizeInBits()); |
0 |
3575 |
KnownBits KnownFull(MemVT.getSizeInBits()); |
0 |
| 3576 |
|
--- |
3576 |
|
--- |
| 3577 |
computeKnownBitsFromRangeMetadata(*Ranges, KnownFull); |
0 |
3577 |
computeKnownBitsFromRangeMetadata(*Ranges, KnownFull); |
0 |
| 3578 |
Known = KnownFull.trunc(BitWidth); |
0 |
3578 |
Known = KnownFull.trunc(BitWidth); |
0 |
| 3579 |
} else |
0 |
3579 |
} else |
0 |
| 3580 |
computeKnownBitsFromRangeMetadata(*Ranges, Known); |
0 |
3580 |
computeKnownBitsFromRangeMetadata(*Ranges, Known); |
0 |
| 3581 |
} |
--- |
3581 |
} |
--- |
| 3582 |
} |
--- |
3582 |
} |
--- |
| 3583 |
break; |
0 |
3583 |
break; |
0 |
| 3584 |
} |
--- |
3584 |
} |
--- |
| 3585 |
case ISD::ZERO_EXTEND_VECTOR_INREG: { |
0 |
3585 |
case ISD::ZERO_EXTEND_VECTOR_INREG: { |
0 |
| 3586 |
if (Op.getValueType().isScalableVector()) |
0 |
3586 |
if (Op.getValueType().isScalableVector()) |
0 |
| 3587 |
break; |
0 |
3587 |
break; |
0 |
| 3588 |
EVT InVT = Op.getOperand(0).getValueType(); |
0 |
3588 |
EVT InVT = Op.getOperand(0).getValueType(); |
0 |
| 3589 |
APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements()); |
0 |
3589 |
APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements()); |
0 |
| 3590 |
Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1); |
0 |
3590 |
Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1); |
0 |
| 3591 |
Known = Known.zext(BitWidth); |
0 |
3591 |
Known = Known.zext(BitWidth); |
0 |
| 3592 |
break; |
0 |
3592 |
break; |
0 |
| 3593 |
} |
0 |
3593 |
} |
0 |
| 3594 |
case ISD::ZERO_EXTEND: { |
0 |
3594 |
case ISD::ZERO_EXTEND: { |
0 |
| 3595 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3595 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3596 |
Known = Known.zext(BitWidth); |
0 |
3596 |
Known = Known.zext(BitWidth); |
0 |
| 3597 |
break; |
0 |
3597 |
break; |
0 |
| 3598 |
} |
--- |
3598 |
} |
--- |
| 3599 |
case ISD::SIGN_EXTEND_VECTOR_INREG: { |
0 |
3599 |
case ISD::SIGN_EXTEND_VECTOR_INREG: { |
0 |
| 3600 |
if (Op.getValueType().isScalableVector()) |
0 |
3600 |
if (Op.getValueType().isScalableVector()) |
0 |
| 3601 |
break; |
0 |
3601 |
break; |
0 |
| 3602 |
EVT InVT = Op.getOperand(0).getValueType(); |
0 |
3602 |
EVT InVT = Op.getOperand(0).getValueType(); |
0 |
| 3603 |
APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements()); |
0 |
3603 |
APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements()); |
0 |
| 3604 |
Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1); |
0 |
3604 |
Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1); |
0 |
| 3605 |
// If the sign bit is known to be zero or one, then sext will extend |
--- |
3605 |
// If the sign bit is known to be zero or one, then sext will extend |
--- |
| 3606 |
// it to the top bits, else it will just zext. |
--- |
3606 |
// it to the top bits, else it will just zext. |
--- |
| 3607 |
Known = Known.sext(BitWidth); |
0 |
3607 |
Known = Known.sext(BitWidth); |
0 |
| 3608 |
break; |
0 |
3608 |
break; |
0 |
| 3609 |
} |
0 |
3609 |
} |
0 |
| 3610 |
case ISD::SIGN_EXTEND: { |
0 |
3610 |
case ISD::SIGN_EXTEND: { |
0 |
| 3611 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3611 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3612 |
// If the sign bit is known to be zero or one, then sext will extend |
--- |
3612 |
// If the sign bit is known to be zero or one, then sext will extend |
--- |
| 3613 |
// it to the top bits, else it will just zext. |
--- |
3613 |
// it to the top bits, else it will just zext. |
--- |
| 3614 |
Known = Known.sext(BitWidth); |
0 |
3614 |
Known = Known.sext(BitWidth); |
0 |
| 3615 |
break; |
0 |
3615 |
break; |
0 |
| 3616 |
} |
--- |
3616 |
} |
--- |
| 3617 |
case ISD::ANY_EXTEND_VECTOR_INREG: { |
0 |
3617 |
case ISD::ANY_EXTEND_VECTOR_INREG: { |
0 |
| 3618 |
if (Op.getValueType().isScalableVector()) |
0 |
3618 |
if (Op.getValueType().isScalableVector()) |
0 |
| 3619 |
break; |
0 |
3619 |
break; |
0 |
| 3620 |
EVT InVT = Op.getOperand(0).getValueType(); |
0 |
3620 |
EVT InVT = Op.getOperand(0).getValueType(); |
0 |
| 3621 |
APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements()); |
0 |
3621 |
APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements()); |
0 |
| 3622 |
Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1); |
0 |
3622 |
Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1); |
0 |
| 3623 |
Known = Known.anyext(BitWidth); |
0 |
3623 |
Known = Known.anyext(BitWidth); |
0 |
| 3624 |
break; |
0 |
3624 |
break; |
0 |
| 3625 |
} |
0 |
3625 |
} |
0 |
| 3626 |
case ISD::ANY_EXTEND: { |
0 |
3626 |
case ISD::ANY_EXTEND: { |
0 |
| 3627 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3627 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3628 |
Known = Known.anyext(BitWidth); |
0 |
3628 |
Known = Known.anyext(BitWidth); |
0 |
| 3629 |
break; |
0 |
3629 |
break; |
0 |
| 3630 |
} |
--- |
3630 |
} |
--- |
| 3631 |
case ISD::TRUNCATE: { |
0 |
3631 |
case ISD::TRUNCATE: { |
0 |
| 3632 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3632 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3633 |
Known = Known.trunc(BitWidth); |
0 |
3633 |
Known = Known.trunc(BitWidth); |
0 |
| 3634 |
break; |
0 |
3634 |
break; |
0 |
| 3635 |
} |
--- |
3635 |
} |
--- |
| 3636 |
case ISD::AssertZext: { |
0 |
3636 |
case ISD::AssertZext: { |
0 |
| 3637 |
EVT VT = cast(Op.getOperand(1))->getVT(); |
0 |
3637 |
EVT VT = cast(Op.getOperand(1))->getVT(); |
0 |
| 3638 |
APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits()); |
0 |
3638 |
APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits()); |
0 |
| 3639 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3639 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3640 |
Known.Zero |= (~InMask); |
0 |
3640 |
Known.Zero |= (~InMask); |
0 |
| 3641 |
Known.One &= (~Known.Zero); |
0 |
3641 |
Known.One &= (~Known.Zero); |
0 |
| 3642 |
break; |
0 |
3642 |
break; |
0 |
| 3643 |
} |
0 |
3643 |
} |
0 |
| 3644 |
case ISD::AssertAlign: { |
0 |
3644 |
case ISD::AssertAlign: { |
0 |
| 3645 |
unsigned LogOfAlign = Log2(cast(Op)->getAlign()); |
0 |
3645 |
unsigned LogOfAlign = Log2(cast(Op)->getAlign()); |
0 |
| 3646 |
assert(LogOfAlign != 0); |
0 |
3646 |
assert(LogOfAlign != 0); |
0 |
| 3647 |
|
--- |
3647 |
|
--- |
| 3648 |
// TODO: Should use maximum with source |
--- |
3648 |
// TODO: Should use maximum with source |
--- |
| 3649 |
// If a node is guaranteed to be aligned, set low zero bits accordingly as |
--- |
3649 |
// If a node is guaranteed to be aligned, set low zero bits accordingly as |
--- |
| 3650 |
// well as clearing one bits. |
--- |
3650 |
// well as clearing one bits. |
--- |
| 3651 |
Known.Zero.setLowBits(LogOfAlign); |
0 |
3651 |
Known.Zero.setLowBits(LogOfAlign); |
0 |
| 3652 |
Known.One.clearLowBits(LogOfAlign); |
0 |
3652 |
Known.One.clearLowBits(LogOfAlign); |
0 |
| 3653 |
break; |
0 |
3653 |
break; |
0 |
| 3654 |
} |
--- |
3654 |
} |
--- |
| 3655 |
case ISD::FGETSIGN: |
0 |
3655 |
case ISD::FGETSIGN: |
0 |
| 3656 |
// All bits are zero except the low bit. |
--- |
3656 |
// All bits are zero except the low bit. |
--- |
| 3657 |
Known.Zero.setBitsFrom(1); |
0 |
3657 |
Known.Zero.setBitsFrom(1); |
0 |
| 3658 |
break; |
0 |
3658 |
break; |
0 |
| 3659 |
case ISD::ADD: |
0 |
3659 |
case ISD::ADD: |
0 |
| 3660 |
case ISD::SUB: { |
--- |
3660 |
case ISD::SUB: { |
--- |
| 3661 |
SDNodeFlags Flags = Op.getNode()->getFlags(); |
0 |
3661 |
SDNodeFlags Flags = Op.getNode()->getFlags(); |
0 |
| 3662 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3662 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3663 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3663 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3664 |
Known = KnownBits::computeForAddSub(Op.getOpcode() == ISD::ADD, |
0 |
3664 |
Known = KnownBits::computeForAddSub(Op.getOpcode() == ISD::ADD, |
0 |
| 3665 |
Flags.hasNoSignedWrap(), Known, Known2); |
0 |
3665 |
Flags.hasNoSignedWrap(), Known, Known2); |
0 |
| 3666 |
break; |
0 |
3666 |
break; |
0 |
| 3667 |
} |
--- |
3667 |
} |
--- |
| 3668 |
case ISD::USUBO: |
0 |
3668 |
case ISD::USUBO: |
0 |
| 3669 |
case ISD::SSUBO: |
--- |
3669 |
case ISD::SSUBO: |
--- |
| 3670 |
case ISD::USUBO_CARRY: |
--- |
3670 |
case ISD::USUBO_CARRY: |
--- |
| 3671 |
case ISD::SSUBO_CARRY: |
--- |
3671 |
case ISD::SSUBO_CARRY: |
--- |
| 3672 |
if (Op.getResNo() == 1) { |
0 |
3672 |
if (Op.getResNo() == 1) { |
0 |
| 3673 |
// If we know the result of a setcc has the top bits zero, use this info. |
--- |
3673 |
// If we know the result of a setcc has the top bits zero, use this info. |
--- |
| 3674 |
if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == |
0 |
3674 |
if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == |
0 |
| 3675 |
TargetLowering::ZeroOrOneBooleanContent && |
0 |
3675 |
TargetLowering::ZeroOrOneBooleanContent && |
0 |
| 3676 |
BitWidth > 1) |
--- |
3676 |
BitWidth > 1) |
--- |
| 3677 |
Known.Zero.setBitsFrom(1); |
0 |
3677 |
Known.Zero.setBitsFrom(1); |
0 |
| 3678 |
break; |
0 |
3678 |
break; |
0 |
| 3679 |
} |
--- |
3679 |
} |
--- |
| 3680 |
[[fallthrough]]; |
--- |
3680 |
[[fallthrough]]; |
--- |
| 3681 |
case ISD::SUBC: { |
--- |
3681 |
case ISD::SUBC: { |
--- |
| 3682 |
assert(Op.getResNo() == 0 && |
0 |
3682 |
assert(Op.getResNo() == 0 && |
0 |
| 3683 |
"We only compute knownbits for the difference here."); |
--- |
3683 |
"We only compute knownbits for the difference here."); |
--- |
| 3684 |
|
--- |
3684 |
|
--- |
| 3685 |
// TODO: Compute influence of the carry operand. |
--- |
3685 |
// TODO: Compute influence of the carry operand. |
--- |
| 3686 |
if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) |
0 |
3686 |
if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) |
0 |
| 3687 |
break; |
--- |
3687 |
break; |
--- |
| 3688 |
|
--- |
3688 |
|
--- |
| 3689 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3689 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3690 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3690 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3691 |
Known = KnownBits::computeForAddSub(/* Add */ false, /* NSW */ false, |
0 |
3691 |
Known = KnownBits::computeForAddSub(/* Add */ false, /* NSW */ false, |
0 |
| 3692 |
Known, Known2); |
0 |
3692 |
Known, Known2); |
0 |
| 3693 |
break; |
0 |
3693 |
break; |
0 |
| 3694 |
} |
--- |
3694 |
} |
--- |
| 3695 |
case ISD::UADDO: |
0 |
3695 |
case ISD::UADDO: |
0 |
| 3696 |
case ISD::SADDO: |
--- |
3696 |
case ISD::SADDO: |
--- |
| 3697 |
case ISD::UADDO_CARRY: |
--- |
3697 |
case ISD::UADDO_CARRY: |
--- |
| 3698 |
case ISD::SADDO_CARRY: |
--- |
3698 |
case ISD::SADDO_CARRY: |
--- |
| 3699 |
if (Op.getResNo() == 1) { |
0 |
3699 |
if (Op.getResNo() == 1) { |
0 |
| 3700 |
// If we know the result of a setcc has the top bits zero, use this info. |
--- |
3700 |
// If we know the result of a setcc has the top bits zero, use this info. |
--- |
| 3701 |
if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == |
0 |
3701 |
if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == |
0 |
| 3702 |
TargetLowering::ZeroOrOneBooleanContent && |
0 |
3702 |
TargetLowering::ZeroOrOneBooleanContent && |
0 |
| 3703 |
BitWidth > 1) |
--- |
3703 |
BitWidth > 1) |
--- |
| 3704 |
Known.Zero.setBitsFrom(1); |
0 |
3704 |
Known.Zero.setBitsFrom(1); |
0 |
| 3705 |
break; |
0 |
3705 |
break; |
0 |
| 3706 |
} |
--- |
3706 |
} |
--- |
| 3707 |
[[fallthrough]]; |
--- |
3707 |
[[fallthrough]]; |
--- |
| 3708 |
case ISD::ADDC: |
--- |
3708 |
case ISD::ADDC: |
--- |
| 3709 |
case ISD::ADDE: { |
--- |
3709 |
case ISD::ADDE: { |
--- |
| 3710 |
assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here."); |
0 |
3710 |
assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here."); |
0 |
| 3711 |
|
--- |
3711 |
|
--- |
| 3712 |
// With ADDE and UADDO_CARRY, a carry bit may be added in. |
--- |
3712 |
// With ADDE and UADDO_CARRY, a carry bit may be added in. |
--- |
| 3713 |
KnownBits Carry(1); |
0 |
3713 |
KnownBits Carry(1); |
0 |
| 3714 |
if (Opcode == ISD::ADDE) |
0 |
3714 |
if (Opcode == ISD::ADDE) |
0 |
| 3715 |
// Can't track carry from glue, set carry to unknown. |
--- |
3715 |
// Can't track carry from glue, set carry to unknown. |
--- |
| 3716 |
Carry.resetAll(); |
0 |
3716 |
Carry.resetAll(); |
0 |
| 3717 |
else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) |
0 |
3717 |
else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) |
0 |
| 3718 |
// TODO: Compute known bits for the carry operand. Not sure if it is worth |
--- |
3718 |
// TODO: Compute known bits for the carry operand. Not sure if it is worth |
--- |
| 3719 |
// the trouble (how often will we find a known carry bit). And I haven't |
--- |
3719 |
// the trouble (how often will we find a known carry bit). And I haven't |
--- |
| 3720 |
// tested this very much yet, but something like this might work: |
--- |
3720 |
// tested this very much yet, but something like this might work: |
--- |
| 3721 |
// Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1); |
--- |
3721 |
// Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1); |
--- |
| 3722 |
// Carry = Carry.zextOrTrunc(1, false); |
--- |
3722 |
// Carry = Carry.zextOrTrunc(1, false); |
--- |
| 3723 |
Carry.resetAll(); |
0 |
3723 |
Carry.resetAll(); |
0 |
| 3724 |
else |
--- |
3724 |
else |
--- |
| 3725 |
Carry.setAllZero(); |
0 |
3725 |
Carry.setAllZero(); |
0 |
| 3726 |
|
--- |
3726 |
|
--- |
| 3727 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3727 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3728 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3728 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3729 |
Known = KnownBits::computeForAddCarry(Known, Known2, Carry); |
0 |
3729 |
Known = KnownBits::computeForAddCarry(Known, Known2, Carry); |
0 |
| 3730 |
break; |
0 |
3730 |
break; |
0 |
| 3731 |
} |
0 |
3731 |
} |
0 |
| 3732 |
case ISD::UDIV: { |
0 |
3732 |
case ISD::UDIV: { |
0 |
| 3733 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3733 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3734 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3734 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3735 |
Known = KnownBits::udiv(Known, Known2, Op->getFlags().hasExact()); |
0 |
3735 |
Known = KnownBits::udiv(Known, Known2, Op->getFlags().hasExact()); |
0 |
| 3736 |
break; |
0 |
3736 |
break; |
0 |
| 3737 |
} |
--- |
3737 |
} |
--- |
| 3738 |
case ISD::SDIV: { |
0 |
3738 |
case ISD::SDIV: { |
0 |
| 3739 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3739 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3740 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3740 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3741 |
Known = KnownBits::sdiv(Known, Known2, Op->getFlags().hasExact()); |
0 |
3741 |
Known = KnownBits::sdiv(Known, Known2, Op->getFlags().hasExact()); |
0 |
| 3742 |
break; |
0 |
3742 |
break; |
0 |
| 3743 |
} |
--- |
3743 |
} |
--- |
| 3744 |
case ISD::SREM: { |
0 |
3744 |
case ISD::SREM: { |
0 |
| 3745 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3745 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3746 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3746 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3747 |
Known = KnownBits::srem(Known, Known2); |
0 |
3747 |
Known = KnownBits::srem(Known, Known2); |
0 |
| 3748 |
break; |
0 |
3748 |
break; |
0 |
| 3749 |
} |
--- |
3749 |
} |
--- |
| 3750 |
case ISD::UREM: { |
0 |
3750 |
case ISD::UREM: { |
0 |
| 3751 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3751 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3752 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3752 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3753 |
Known = KnownBits::urem(Known, Known2); |
0 |
3753 |
Known = KnownBits::urem(Known, Known2); |
0 |
| 3754 |
break; |
0 |
3754 |
break; |
0 |
| 3755 |
} |
--- |
3755 |
} |
--- |
| 3756 |
case ISD::EXTRACT_ELEMENT: { |
0 |
3756 |
case ISD::EXTRACT_ELEMENT: { |
0 |
| 3757 |
Known = computeKnownBits(Op.getOperand(0), Depth+1); |
0 |
3757 |
Known = computeKnownBits(Op.getOperand(0), Depth+1); |
0 |
| 3758 |
const unsigned Index = Op.getConstantOperandVal(1); |
0 |
3758 |
const unsigned Index = Op.getConstantOperandVal(1); |
0 |
| 3759 |
const unsigned EltBitWidth = Op.getValueSizeInBits(); |
0 |
3759 |
const unsigned EltBitWidth = Op.getValueSizeInBits(); |
0 |
| 3760 |
|
--- |
3760 |
|
--- |
| 3761 |
// Remove low part of known bits mask |
--- |
3761 |
// Remove low part of known bits mask |
--- |
| 3762 |
Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth); |
0 |
3762 |
Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth); |
0 |
| 3763 |
Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth); |
0 |
3763 |
Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth); |
0 |
| 3764 |
|
--- |
3764 |
|
--- |
| 3765 |
// Remove high part of known bit mask |
--- |
3765 |
// Remove high part of known bit mask |
--- |
| 3766 |
Known = Known.trunc(EltBitWidth); |
0 |
3766 |
Known = Known.trunc(EltBitWidth); |
0 |
| 3767 |
break; |
0 |
3767 |
break; |
0 |
| 3768 |
} |
--- |
3768 |
} |
--- |
| 3769 |
case ISD::EXTRACT_VECTOR_ELT: { |
0 |
3769 |
case ISD::EXTRACT_VECTOR_ELT: { |
0 |
| 3770 |
SDValue InVec = Op.getOperand(0); |
0 |
3770 |
SDValue InVec = Op.getOperand(0); |
0 |
| 3771 |
SDValue EltNo = Op.getOperand(1); |
0 |
3771 |
SDValue EltNo = Op.getOperand(1); |
0 |
| 3772 |
EVT VecVT = InVec.getValueType(); |
0 |
3772 |
EVT VecVT = InVec.getValueType(); |
0 |
| 3773 |
// computeKnownBits not yet implemented for scalable vectors. |
--- |
3773 |
// computeKnownBits not yet implemented for scalable vectors. |
--- |
| 3774 |
if (VecVT.isScalableVector()) |
0 |
3774 |
if (VecVT.isScalableVector()) |
0 |
| 3775 |
break; |
0 |
3775 |
break; |
0 |
| 3776 |
const unsigned EltBitWidth = VecVT.getScalarSizeInBits(); |
0 |
3776 |
const unsigned EltBitWidth = VecVT.getScalarSizeInBits(); |
0 |
| 3777 |
const unsigned NumSrcElts = VecVT.getVectorNumElements(); |
0 |
3777 |
const unsigned NumSrcElts = VecVT.getVectorNumElements(); |
0 |
| 3778 |
|
--- |
3778 |
|
--- |
| 3779 |
// If BitWidth > EltBitWidth the value is anyext:ed. So we do not know |
--- |
3779 |
// If BitWidth > EltBitWidth the value is anyext:ed. So we do not know |
--- |
| 3780 |
// anything about the extended bits. |
--- |
3780 |
// anything about the extended bits. |
--- |
| 3781 |
if (BitWidth > EltBitWidth) |
0 |
3781 |
if (BitWidth > EltBitWidth) |
0 |
| 3782 |
Known = Known.trunc(EltBitWidth); |
0 |
3782 |
Known = Known.trunc(EltBitWidth); |
0 |
| 3783 |
|
--- |
3783 |
|
--- |
| 3784 |
// If we know the element index, just demand that vector element, else for |
--- |
3784 |
// If we know the element index, just demand that vector element, else for |
--- |
| 3785 |
// an unknown element index, ignore DemandedElts and demand them all. |
--- |
3785 |
// an unknown element index, ignore DemandedElts and demand them all. |
--- |
| 3786 |
APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts); |
0 |
3786 |
APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts); |
0 |
| 3787 |
auto *ConstEltNo = dyn_cast(EltNo); |
0 |
3787 |
auto *ConstEltNo = dyn_cast(EltNo); |
0 |
| 3788 |
if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts)) |
0 |
3788 |
if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts)) |
0 |
| 3789 |
DemandedSrcElts = |
--- |
3789 |
DemandedSrcElts = |
--- |
| 3790 |
APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue()); |
0 |
3790 |
APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue()); |
0 |
| 3791 |
|
--- |
3791 |
|
--- |
| 3792 |
Known = computeKnownBits(InVec, DemandedSrcElts, Depth + 1); |
0 |
3792 |
Known = computeKnownBits(InVec, DemandedSrcElts, Depth + 1); |
0 |
| 3793 |
if (BitWidth > EltBitWidth) |
0 |
3793 |
if (BitWidth > EltBitWidth) |
0 |
| 3794 |
Known = Known.anyext(BitWidth); |
0 |
3794 |
Known = Known.anyext(BitWidth); |
0 |
| 3795 |
break; |
0 |
3795 |
break; |
0 |
| 3796 |
} |
0 |
3796 |
} |
0 |
| 3797 |
case ISD::INSERT_VECTOR_ELT: { |
0 |
3797 |
case ISD::INSERT_VECTOR_ELT: { |
0 |
| 3798 |
if (Op.getValueType().isScalableVector()) |
0 |
3798 |
if (Op.getValueType().isScalableVector()) |
0 |
| 3799 |
break; |
0 |
3799 |
break; |
0 |
| 3800 |
|
--- |
3800 |
|
--- |
| 3801 |
// If we know the element index, split the demand between the |
--- |
3801 |
// If we know the element index, split the demand between the |
--- |
| 3802 |
// source vector and the inserted element, otherwise assume we need |
--- |
3802 |
// source vector and the inserted element, otherwise assume we need |
--- |
| 3803 |
// the original demanded vector elements and the value. |
--- |
3803 |
// the original demanded vector elements and the value. |
--- |
| 3804 |
SDValue InVec = Op.getOperand(0); |
0 |
3804 |
SDValue InVec = Op.getOperand(0); |
0 |
| 3805 |
SDValue InVal = Op.getOperand(1); |
0 |
3805 |
SDValue InVal = Op.getOperand(1); |
0 |
| 3806 |
SDValue EltNo = Op.getOperand(2); |
0 |
3806 |
SDValue EltNo = Op.getOperand(2); |
0 |
| 3807 |
bool DemandedVal = true; |
0 |
3807 |
bool DemandedVal = true; |
0 |
| 3808 |
APInt DemandedVecElts = DemandedElts; |
0 |
3808 |
APInt DemandedVecElts = DemandedElts; |
0 |
| 3809 |
auto *CEltNo = dyn_cast(EltNo); |
0 |
3809 |
auto *CEltNo = dyn_cast(EltNo); |
0 |
| 3810 |
if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) { |
0 |
3810 |
if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) { |
0 |
| 3811 |
unsigned EltIdx = CEltNo->getZExtValue(); |
0 |
3811 |
unsigned EltIdx = CEltNo->getZExtValue(); |
0 |
| 3812 |
DemandedVal = !!DemandedElts[EltIdx]; |
0 |
3812 |
DemandedVal = !!DemandedElts[EltIdx]; |
0 |
| 3813 |
DemandedVecElts.clearBit(EltIdx); |
0 |
3813 |
DemandedVecElts.clearBit(EltIdx); |
0 |
| 3814 |
} |
--- |
3814 |
} |
--- |
| 3815 |
Known.One.setAllBits(); |
0 |
3815 |
Known.One.setAllBits(); |
0 |
| 3816 |
Known.Zero.setAllBits(); |
0 |
3816 |
Known.Zero.setAllBits(); |
0 |
| 3817 |
if (DemandedVal) { |
0 |
3817 |
if (DemandedVal) { |
0 |
| 3818 |
Known2 = computeKnownBits(InVal, Depth + 1); |
0 |
3818 |
Known2 = computeKnownBits(InVal, Depth + 1); |
0 |
| 3819 |
Known = Known.intersectWith(Known2.zextOrTrunc(BitWidth)); |
0 |
3819 |
Known = Known.intersectWith(Known2.zextOrTrunc(BitWidth)); |
0 |
| 3820 |
} |
--- |
3820 |
} |
--- |
| 3821 |
if (!!DemandedVecElts) { |
0 |
3821 |
if (!!DemandedVecElts) { |
0 |
| 3822 |
Known2 = computeKnownBits(InVec, DemandedVecElts, Depth + 1); |
0 |
3822 |
Known2 = computeKnownBits(InVec, DemandedVecElts, Depth + 1); |
0 |
| 3823 |
Known = Known.intersectWith(Known2); |
0 |
3823 |
Known = Known.intersectWith(Known2); |
0 |
| 3824 |
} |
--- |
3824 |
} |
--- |
| 3825 |
break; |
0 |
3825 |
break; |
0 |
| 3826 |
} |
0 |
3826 |
} |
0 |
| 3827 |
case ISD::BITREVERSE: { |
0 |
3827 |
case ISD::BITREVERSE: { |
0 |
| 3828 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3828 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3829 |
Known = Known2.reverseBits(); |
0 |
3829 |
Known = Known2.reverseBits(); |
0 |
| 3830 |
break; |
0 |
3830 |
break; |
0 |
| 3831 |
} |
--- |
3831 |
} |
--- |
| 3832 |
case ISD::BSWAP: { |
0 |
3832 |
case ISD::BSWAP: { |
0 |
| 3833 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3833 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3834 |
Known = Known2.byteSwap(); |
0 |
3834 |
Known = Known2.byteSwap(); |
0 |
| 3835 |
break; |
0 |
3835 |
break; |
0 |
| 3836 |
} |
--- |
3836 |
} |
--- |
| 3837 |
case ISD::ABS: { |
0 |
3837 |
case ISD::ABS: { |
0 |
| 3838 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3838 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3839 |
Known = Known2.abs(); |
0 |
3839 |
Known = Known2.abs(); |
0 |
| 3840 |
break; |
0 |
3840 |
break; |
0 |
| 3841 |
} |
--- |
3841 |
} |
--- |
| 3842 |
case ISD::USUBSAT: { |
0 |
3842 |
case ISD::USUBSAT: { |
0 |
| 3843 |
// The result of usubsat will never be larger than the LHS. |
--- |
3843 |
// The result of usubsat will never be larger than the LHS. |
--- |
| 3844 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3844 |
Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3845 |
Known.Zero.setHighBits(Known2.countMinLeadingZeros()); |
0 |
3845 |
Known.Zero.setHighBits(Known2.countMinLeadingZeros()); |
0 |
| 3846 |
break; |
0 |
3846 |
break; |
0 |
| 3847 |
} |
--- |
3847 |
} |
--- |
| 3848 |
case ISD::UMIN: { |
0 |
3848 |
case ISD::UMIN: { |
0 |
| 3849 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3849 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3850 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3850 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3851 |
Known = KnownBits::umin(Known, Known2); |
0 |
3851 |
Known = KnownBits::umin(Known, Known2); |
0 |
| 3852 |
break; |
0 |
3852 |
break; |
0 |
| 3853 |
} |
--- |
3853 |
} |
--- |
| 3854 |
case ISD::UMAX: { |
0 |
3854 |
case ISD::UMAX: { |
0 |
| 3855 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3855 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3856 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3856 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3857 |
Known = KnownBits::umax(Known, Known2); |
0 |
3857 |
Known = KnownBits::umax(Known, Known2); |
0 |
| 3858 |
break; |
0 |
3858 |
break; |
0 |
| 3859 |
} |
--- |
3859 |
} |
--- |
| 3860 |
case ISD::SMIN: |
0 |
3860 |
case ISD::SMIN: |
0 |
| 3861 |
case ISD::SMAX: { |
--- |
3861 |
case ISD::SMAX: { |
--- |
| 3862 |
// If we have a clamp pattern, we know that the number of sign bits will be |
--- |
3862 |
// If we have a clamp pattern, we know that the number of sign bits will be |
--- |
| 3863 |
// the minimum of the clamp min/max range. |
--- |
3863 |
// the minimum of the clamp min/max range. |
--- |
| 3864 |
bool IsMax = (Opcode == ISD::SMAX); |
0 |
3864 |
bool IsMax = (Opcode == ISD::SMAX); |
0 |
| 3865 |
ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr; |
0 |
3865 |
ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr; |
0 |
| 3866 |
if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts))) |
0 |
3866 |
if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts))) |
0 |
| 3867 |
if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) |
0 |
3867 |
if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) |
0 |
| 3868 |
CstHigh = |
0 |
3868 |
CstHigh = |
0 |
| 3869 |
isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts); |
0 |
3869 |
isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts); |
0 |
| 3870 |
if (CstLow && CstHigh) { |
0 |
3870 |
if (CstLow && CstHigh) { |
0 |
| 3871 |
if (!IsMax) |
0 |
3871 |
if (!IsMax) |
0 |
| 3872 |
std::swap(CstLow, CstHigh); |
0 |
3872 |
std::swap(CstLow, CstHigh); |
0 |
| 3873 |
|
--- |
3873 |
|
--- |
| 3874 |
const APInt &ValueLow = CstLow->getAPIntValue(); |
0 |
3874 |
const APInt &ValueLow = CstLow->getAPIntValue(); |
0 |
| 3875 |
const APInt &ValueHigh = CstHigh->getAPIntValue(); |
0 |
3875 |
const APInt &ValueHigh = CstHigh->getAPIntValue(); |
0 |
| 3876 |
if (ValueLow.sle(ValueHigh)) { |
0 |
3876 |
if (ValueLow.sle(ValueHigh)) { |
0 |
| 3877 |
unsigned LowSignBits = ValueLow.getNumSignBits(); |
0 |
3877 |
unsigned LowSignBits = ValueLow.getNumSignBits(); |
0 |
| 3878 |
unsigned HighSignBits = ValueHigh.getNumSignBits(); |
0 |
3878 |
unsigned HighSignBits = ValueHigh.getNumSignBits(); |
0 |
| 3879 |
unsigned MinSignBits = std::min(LowSignBits, HighSignBits); |
0 |
3879 |
unsigned MinSignBits = std::min(LowSignBits, HighSignBits); |
0 |
| 3880 |
if (ValueLow.isNegative() && ValueHigh.isNegative()) { |
0 |
3880 |
if (ValueLow.isNegative() && ValueHigh.isNegative()) { |
0 |
| 3881 |
Known.One.setHighBits(MinSignBits); |
0 |
3881 |
Known.One.setHighBits(MinSignBits); |
0 |
| 3882 |
break; |
0 |
3882 |
break; |
0 |
| 3883 |
} |
--- |
3883 |
} |
--- |
| 3884 |
if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) { |
0 |
3884 |
if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) { |
0 |
| 3885 |
Known.Zero.setHighBits(MinSignBits); |
0 |
3885 |
Known.Zero.setHighBits(MinSignBits); |
0 |
| 3886 |
break; |
0 |
3886 |
break; |
0 |
| 3887 |
} |
--- |
3887 |
} |
--- |
| 3888 |
} |
--- |
3888 |
} |
--- |
| 3889 |
} |
--- |
3889 |
} |
--- |
| 3890 |
|
--- |
3890 |
|
--- |
| 3891 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
3891 |
Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 3892 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
3892 |
Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 3893 |
if (IsMax) |
0 |
3893 |
if (IsMax) |
0 |
| 3894 |
Known = KnownBits::smax(Known, Known2); |
0 |
3894 |
Known = KnownBits::smax(Known, Known2); |
0 |
| 3895 |
else |
--- |
3895 |
else |
--- |
| 3896 |
Known = KnownBits::smin(Known, Known2); |
0 |
3896 |
Known = KnownBits::smin(Known, Known2); |
0 |
| 3897 |
|
--- |
3897 |
|
--- |
| 3898 |
// For SMAX, if CstLow is non-negative we know the result will be |
--- |
3898 |
// For SMAX, if CstLow is non-negative we know the result will be |
--- |
| 3899 |
// non-negative and thus all sign bits are 0. |
--- |
3899 |
// non-negative and thus all sign bits are 0. |
--- |
| 3900 |
// TODO: There's an equivalent of this for smin with negative constant for |
--- |
3900 |
// TODO: There's an equivalent of this for smin with negative constant for |
--- |
| 3901 |
// known ones. |
--- |
3901 |
// known ones. |
--- |
| 3902 |
if (IsMax && CstLow) { |
0 |
3902 |
if (IsMax && CstLow) { |
0 |
| 3903 |
const APInt &ValueLow = CstLow->getAPIntValue(); |
0 |
3903 |
const APInt &ValueLow = CstLow->getAPIntValue(); |
0 |
| 3904 |
if (ValueLow.isNonNegative()) { |
0 |
3904 |
if (ValueLow.isNonNegative()) { |
0 |
| 3905 |
unsigned SignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1); |
0 |
3905 |
unsigned SignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1); |
0 |
| 3906 |
Known.Zero.setHighBits(std::min(SignBits, ValueLow.getNumSignBits())); |
0 |
3906 |
Known.Zero.setHighBits(std::min(SignBits, ValueLow.getNumSignBits())); |
0 |
| 3907 |
} |
--- |
3907 |
} |
--- |
| 3908 |
} |
--- |
3908 |
} |
--- |
| 3909 |
|
--- |
3909 |
|
--- |
| 3910 |
break; |
0 |
3910 |
break; |
0 |
| 3911 |
} |
--- |
3911 |
} |
--- |
| 3912 |
case ISD::FP_TO_UINT_SAT: { |
0 |
3912 |
case ISD::FP_TO_UINT_SAT: { |
0 |
| 3913 |
// FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT. |
--- |
3913 |
// FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT. |
--- |
| 3914 |
EVT VT = cast(Op.getOperand(1))->getVT(); |
0 |
3914 |
EVT VT = cast(Op.getOperand(1))->getVT(); |
0 |
| 3915 |
Known.Zero |= APInt::getBitsSetFrom(BitWidth, VT.getScalarSizeInBits()); |
0 |
3915 |
Known.Zero |= APInt::getBitsSetFrom(BitWidth, VT.getScalarSizeInBits()); |
0 |
| 3916 |
break; |
0 |
3916 |
break; |
0 |
| 3917 |
} |
--- |
3917 |
} |
--- |
| 3918 |
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: |
0 |
3918 |
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: |
0 |
| 3919 |
if (Op.getResNo() == 1) { |
0 |
3919 |
if (Op.getResNo() == 1) { |
0 |
| 3920 |
// The boolean result conforms to getBooleanContents. |
--- |
3920 |
// The boolean result conforms to getBooleanContents. |
--- |
| 3921 |
// If we know the result of a setcc has the top bits zero, use this info. |
--- |
3921 |
// If we know the result of a setcc has the top bits zero, use this info. |
--- |
| 3922 |
// We know that we have an integer-based boolean since these operations |
--- |
3922 |
// We know that we have an integer-based boolean since these operations |
--- |
| 3923 |
// are only available for integer. |
--- |
3923 |
// are only available for integer. |
--- |
| 3924 |
if (TLI->getBooleanContents(Op.getValueType().isVector(), false) == |
0 |
3924 |
if (TLI->getBooleanContents(Op.getValueType().isVector(), false) == |
0 |
| 3925 |
TargetLowering::ZeroOrOneBooleanContent && |
0 |
3925 |
TargetLowering::ZeroOrOneBooleanContent && |
0 |
| 3926 |
BitWidth > 1) |
--- |
3926 |
BitWidth > 1) |
--- |
| 3927 |
Known.Zero.setBitsFrom(1); |
0 |
3927 |
Known.Zero.setBitsFrom(1); |
0 |
| 3928 |
break; |
0 |
3928 |
break; |
0 |
| 3929 |
} |
--- |
3929 |
} |
--- |
| 3930 |
[[fallthrough]]; |
--- |
3930 |
[[fallthrough]]; |
--- |
| 3931 |
case ISD::ATOMIC_CMP_SWAP: |
--- |
3931 |
case ISD::ATOMIC_CMP_SWAP: |
--- |
| 3932 |
case ISD::ATOMIC_SWAP: |
--- |
3932 |
case ISD::ATOMIC_SWAP: |
--- |
| 3933 |
case ISD::ATOMIC_LOAD_ADD: |
--- |
3933 |
case ISD::ATOMIC_LOAD_ADD: |
--- |
| 3934 |
case ISD::ATOMIC_LOAD_SUB: |
--- |
3934 |
case ISD::ATOMIC_LOAD_SUB: |
--- |
| 3935 |
case ISD::ATOMIC_LOAD_AND: |
--- |
3935 |
case ISD::ATOMIC_LOAD_AND: |
--- |
| 3936 |
case ISD::ATOMIC_LOAD_CLR: |
--- |
3936 |
case ISD::ATOMIC_LOAD_CLR: |
--- |
| 3937 |
case ISD::ATOMIC_LOAD_OR: |
--- |
3937 |
case ISD::ATOMIC_LOAD_OR: |
--- |
| 3938 |
case ISD::ATOMIC_LOAD_XOR: |
--- |
3938 |
case ISD::ATOMIC_LOAD_XOR: |
--- |
| 3939 |
case ISD::ATOMIC_LOAD_NAND: |
--- |
3939 |
case ISD::ATOMIC_LOAD_NAND: |
--- |
| 3940 |
case ISD::ATOMIC_LOAD_MIN: |
--- |
3940 |
case ISD::ATOMIC_LOAD_MIN: |
--- |
| 3941 |
case ISD::ATOMIC_LOAD_MAX: |
--- |
3941 |
case ISD::ATOMIC_LOAD_MAX: |
--- |
| 3942 |
case ISD::ATOMIC_LOAD_UMIN: |
--- |
3942 |
case ISD::ATOMIC_LOAD_UMIN: |
--- |
| 3943 |
case ISD::ATOMIC_LOAD_UMAX: |
--- |
3943 |
case ISD::ATOMIC_LOAD_UMAX: |
--- |
| 3944 |
case ISD::ATOMIC_LOAD: { |
--- |
3944 |
case ISD::ATOMIC_LOAD: { |
--- |
| 3945 |
unsigned MemBits = |
--- |
3945 |
unsigned MemBits = |
--- |
| 3946 |
cast(Op)->getMemoryVT().getScalarSizeInBits(); |
0 |
3946 |
cast(Op)->getMemoryVT().getScalarSizeInBits(); |
0 |
| 3947 |
// If we are looking at the loaded value. |
--- |
3947 |
// If we are looking at the loaded value. |
--- |
| 3948 |
if (Op.getResNo() == 0) { |
0 |
3948 |
if (Op.getResNo() == 0) { |
0 |
| 3949 |
if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND) |
0 |
3949 |
if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND) |
0 |
| 3950 |
Known.Zero.setBitsFrom(MemBits); |
0 |
3950 |
Known.Zero.setBitsFrom(MemBits); |
0 |
| 3951 |
} |
--- |
3951 |
} |
--- |
| 3952 |
break; |
0 |
3952 |
break; |
0 |
| 3953 |
} |
--- |
3953 |
} |
--- |
| 3954 |
case ISD::FrameIndex: |
0 |
3954 |
case ISD::FrameIndex: |
0 |
| 3955 |
case ISD::TargetFrameIndex: |
--- |
3955 |
case ISD::TargetFrameIndex: |
--- |
| 3956 |
TLI->computeKnownBitsForFrameIndex(cast(Op)->getIndex(), |
0 |
3956 |
TLI->computeKnownBitsForFrameIndex(cast(Op)->getIndex(), |
0 |
| 3957 |
Known, getMachineFunction()); |
0 |
3957 |
Known, getMachineFunction()); |
0 |
| 3958 |
break; |
0 |
3958 |
break; |
0 |
| 3959 |
|
--- |
3959 |
|
--- |
| 3960 |
default: |
0 |
3960 |
default: |
0 |
| 3961 |
if (Opcode < ISD::BUILTIN_OP_END) |
0 |
3961 |
if (Opcode < ISD::BUILTIN_OP_END) |
0 |
| 3962 |
break; |
0 |
3962 |
break; |
0 |
| 3963 |
[[fallthrough]]; |
--- |
3963 |
[[fallthrough]]; |
--- |
| 3964 |
case ISD::INTRINSIC_WO_CHAIN: |
--- |
3964 |
case ISD::INTRINSIC_WO_CHAIN: |
--- |
| 3965 |
case ISD::INTRINSIC_W_CHAIN: |
--- |
3965 |
case ISD::INTRINSIC_W_CHAIN: |
--- |
| 3966 |
case ISD::INTRINSIC_VOID: |
--- |
3966 |
case ISD::INTRINSIC_VOID: |
--- |
| 3967 |
// TODO: Probably okay to remove after audit; here to reduce change size |
--- |
3967 |
// TODO: Probably okay to remove after audit; here to reduce change size |
--- |
| 3968 |
// in initial enablement patch for scalable vectors |
--- |
3968 |
// in initial enablement patch for scalable vectors |
--- |
| 3969 |
if (Op.getValueType().isScalableVector()) |
0 |
3969 |
if (Op.getValueType().isScalableVector()) |
0 |
| 3970 |
break; |
0 |
3970 |
break; |
0 |
| 3971 |
|
--- |
3971 |
|
--- |
| 3972 |
// Allow the target to implement this method for its nodes. |
--- |
3972 |
// Allow the target to implement this method for its nodes. |
--- |
| 3973 |
TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth); |
0 |
3973 |
TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth); |
0 |
| 3974 |
break; |
0 |
3974 |
break; |
0 |
| 3975 |
} |
--- |
3975 |
} |
--- |
| 3976 |
|
--- |
3976 |
|
--- |
| 3977 |
assert(!Known.hasConflict() && "Bits known to be one AND zero?"); |
4 |
3977 |
assert(!Known.hasConflict() && "Bits known to be one AND zero?"); |
4 |
| 3978 |
return Known; |
4 |
3978 |
return Known; |
4 |
| 3979 |
} |
6 |
3979 |
} |
6 |
| 3980 |
|
--- |
3980 |
|
--- |
| 3981 |
/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind. |
--- |
3981 |
/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind. |
--- |
| 3982 |
static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) { |
0 |
3982 |
static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) { |
0 |
| 3983 |
switch (OR) { |
0 |
3983 |
switch (OR) { |
0 |
| 3984 |
case ConstantRange::OverflowResult::MayOverflow: |
0 |
3984 |
case ConstantRange::OverflowResult::MayOverflow: |
0 |
| 3985 |
return SelectionDAG::OFK_Sometime; |
0 |
3985 |
return SelectionDAG::OFK_Sometime; |
0 |
| 3986 |
case ConstantRange::OverflowResult::AlwaysOverflowsLow: |
0 |
3986 |
case ConstantRange::OverflowResult::AlwaysOverflowsLow: |
0 |
| 3987 |
case ConstantRange::OverflowResult::AlwaysOverflowsHigh: |
--- |
3987 |
case ConstantRange::OverflowResult::AlwaysOverflowsHigh: |
--- |
| 3988 |
return SelectionDAG::OFK_Always; |
0 |
3988 |
return SelectionDAG::OFK_Always; |
0 |
| 3989 |
case ConstantRange::OverflowResult::NeverOverflows: |
0 |
3989 |
case ConstantRange::OverflowResult::NeverOverflows: |
0 |
| 3990 |
return SelectionDAG::OFK_Never; |
0 |
3990 |
return SelectionDAG::OFK_Never; |
0 |
| 3991 |
} |
--- |
3991 |
} |
--- |
| 3992 |
llvm_unreachable("Unknown OverflowResult"); |
0 |
3992 |
llvm_unreachable("Unknown OverflowResult"); |
0 |
| 3993 |
} |
--- |
3993 |
} |
--- |
| 3994 |
|
--- |
3994 |
|
--- |
| 3995 |
SelectionDAG::OverflowKind |
--- |
3995 |
SelectionDAG::OverflowKind |
--- |
| 3996 |
SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const { |
0 |
3996 |
SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const { |
0 |
| 3997 |
// X + 0 never overflow |
--- |
3997 |
// X + 0 never overflow |
--- |
| 3998 |
if (isNullConstant(N1)) |
0 |
3998 |
if (isNullConstant(N1)) |
0 |
| 3999 |
return OFK_Never; |
0 |
3999 |
return OFK_Never; |
0 |
| 4000 |
|
--- |
4000 |
|
--- |
| 4001 |
// If both operands each have at least two sign bits, the addition |
--- |
4001 |
// If both operands each have at least two sign bits, the addition |
--- |
| 4002 |
// cannot overflow. |
--- |
4002 |
// cannot overflow. |
--- |
| 4003 |
if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1) |
0 |
4003 |
if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1) |
0 |
| 4004 |
return OFK_Never; |
0 |
4004 |
return OFK_Never; |
0 |
| 4005 |
|
--- |
4005 |
|
--- |
| 4006 |
// TODO: Add ConstantRange::signedAddMayOverflow handling. |
--- |
4006 |
// TODO: Add ConstantRange::signedAddMayOverflow handling. |
--- |
| 4007 |
return OFK_Sometime; |
0 |
4007 |
return OFK_Sometime; |
0 |
| 4008 |
} |
--- |
4008 |
} |
--- |
| 4009 |
|
--- |
4009 |
|
--- |
| 4010 |
SelectionDAG::OverflowKind |
--- |
4010 |
SelectionDAG::OverflowKind |
--- |
| 4011 |
SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const { |
0 |
4011 |
SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const { |
0 |
| 4012 |
// X + 0 never overflow |
--- |
4012 |
// X + 0 never overflow |
--- |
| 4013 |
if (isNullConstant(N1)) |
0 |
4013 |
if (isNullConstant(N1)) |
0 |
| 4014 |
return OFK_Never; |
0 |
4014 |
return OFK_Never; |
0 |
| 4015 |
|
--- |
4015 |
|
--- |
| 4016 |
// mulhi + 1 never overflow |
--- |
4016 |
// mulhi + 1 never overflow |
--- |
| 4017 |
KnownBits N1Known = computeKnownBits(N1); |
0 |
4017 |
KnownBits N1Known = computeKnownBits(N1); |
0 |
| 4018 |
if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 && |
0 |
4018 |
if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 && |
0 |
| 4019 |
N1Known.getMaxValue().ult(2)) |
0 |
4019 |
N1Known.getMaxValue().ult(2)) |
0 |
| 4020 |
return OFK_Never; |
0 |
4020 |
return OFK_Never; |
0 |
| 4021 |
|
--- |
4021 |
|
--- |
| 4022 |
KnownBits N0Known = computeKnownBits(N0); |
0 |
4022 |
KnownBits N0Known = computeKnownBits(N0); |
0 |
| 4023 |
if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 && |
0 |
4023 |
if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 && |
0 |
| 4024 |
N0Known.getMaxValue().ult(2)) |
0 |
4024 |
N0Known.getMaxValue().ult(2)) |
0 |
| 4025 |
return OFK_Never; |
0 |
4025 |
return OFK_Never; |
0 |
| 4026 |
|
--- |
4026 |
|
--- |
| 4027 |
// Fallback to ConstantRange::unsignedAddMayOverflow handling. |
--- |
4027 |
// Fallback to ConstantRange::unsignedAddMayOverflow handling. |
--- |
| 4028 |
ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false); |
0 |
4028 |
ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false); |
0 |
| 4029 |
ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false); |
0 |
4029 |
ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false); |
0 |
| 4030 |
return mapOverflowResult(N0Range.unsignedAddMayOverflow(N1Range)); |
0 |
4030 |
return mapOverflowResult(N0Range.unsignedAddMayOverflow(N1Range)); |
0 |
| 4031 |
} |
0 |
4031 |
} |
0 |
| 4032 |
|
--- |
4032 |
|
--- |
| 4033 |
SelectionDAG::OverflowKind |
--- |
4033 |
SelectionDAG::OverflowKind |
--- |
| 4034 |
SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const { |
0 |
4034 |
SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const { |
0 |
| 4035 |
// X - 0 never overflow |
--- |
4035 |
// X - 0 never overflow |
--- |
| 4036 |
if (isNullConstant(N1)) |
0 |
4036 |
if (isNullConstant(N1)) |
0 |
| 4037 |
return OFK_Never; |
0 |
4037 |
return OFK_Never; |
0 |
| 4038 |
|
--- |
4038 |
|
--- |
| 4039 |
// If both operands each have at least two sign bits, the subtraction |
--- |
4039 |
// If both operands each have at least two sign bits, the subtraction |
--- |
| 4040 |
// cannot overflow. |
--- |
4040 |
// cannot overflow. |
--- |
| 4041 |
if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1) |
0 |
4041 |
if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1) |
0 |
| 4042 |
return OFK_Never; |
0 |
4042 |
return OFK_Never; |
0 |
| 4043 |
|
--- |
4043 |
|
--- |
| 4044 |
// TODO: Add ConstantRange::signedSubMayOverflow handling. |
--- |
4044 |
// TODO: Add ConstantRange::signedSubMayOverflow handling. |
--- |
| 4045 |
return OFK_Sometime; |
0 |
4045 |
return OFK_Sometime; |
0 |
| 4046 |
} |
--- |
4046 |
} |
--- |
| 4047 |
|
--- |
4047 |
|
--- |
| 4048 |
SelectionDAG::OverflowKind |
--- |
4048 |
SelectionDAG::OverflowKind |
--- |
| 4049 |
SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const { |
0 |
4049 |
SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const { |
0 |
| 4050 |
// X - 0 never overflow |
--- |
4050 |
// X - 0 never overflow |
--- |
| 4051 |
if (isNullConstant(N1)) |
0 |
4051 |
if (isNullConstant(N1)) |
0 |
| 4052 |
return OFK_Never; |
0 |
4052 |
return OFK_Never; |
0 |
| 4053 |
|
--- |
4053 |
|
--- |
| 4054 |
// TODO: Add ConstantRange::unsignedSubMayOverflow handling. |
--- |
4054 |
// TODO: Add ConstantRange::unsignedSubMayOverflow handling. |
--- |
| 4055 |
return OFK_Sometime; |
0 |
4055 |
return OFK_Sometime; |
0 |
| 4056 |
} |
--- |
4056 |
} |
--- |
| 4057 |
|
--- |
4057 |
|
--- |
| 4058 |
bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth) const { |
0 |
4058 |
bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth) const { |
0 |
| 4059 |
if (Depth >= MaxRecursionDepth) |
0 |
4059 |
if (Depth >= MaxRecursionDepth) |
0 |
| 4060 |
return false; // Limit search depth. |
0 |
4060 |
return false; // Limit search depth. |
0 |
| 4061 |
|
--- |
4061 |
|
--- |
| 4062 |
EVT OpVT = Val.getValueType(); |
0 |
4062 |
EVT OpVT = Val.getValueType(); |
0 |
| 4063 |
unsigned BitWidth = OpVT.getScalarSizeInBits(); |
0 |
4063 |
unsigned BitWidth = OpVT.getScalarSizeInBits(); |
0 |
| 4064 |
|
--- |
4064 |
|
--- |
| 4065 |
// Is the constant a known power of 2? |
--- |
4065 |
// Is the constant a known power of 2? |
--- |
| 4066 |
if (ConstantSDNode *Const = dyn_cast(Val)) |
0 |
4066 |
if (ConstantSDNode *Const = dyn_cast(Val)) |
0 |
| 4067 |
return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2(); |
0 |
4067 |
return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2(); |
0 |
| 4068 |
|
--- |
4068 |
|
--- |
| 4069 |
// A left-shift of a constant one will have exactly one bit set because |
--- |
4069 |
// A left-shift of a constant one will have exactly one bit set because |
--- |
| 4070 |
// shifting the bit off the end is undefined. |
--- |
4070 |
// shifting the bit off the end is undefined. |
--- |
| 4071 |
if (Val.getOpcode() == ISD::SHL) { |
0 |
4071 |
if (Val.getOpcode() == ISD::SHL) { |
0 |
| 4072 |
auto *C = isConstOrConstSplat(Val.getOperand(0)); |
0 |
4072 |
auto *C = isConstOrConstSplat(Val.getOperand(0)); |
0 |
| 4073 |
if (C && C->getAPIntValue() == 1) |
0 |
4073 |
if (C && C->getAPIntValue() == 1) |
0 |
| 4074 |
return true; |
0 |
4074 |
return true; |
0 |
| 4075 |
} |
--- |
4075 |
} |
--- |
| 4076 |
|
--- |
4076 |
|
--- |
| 4077 |
// Similarly, a logical right-shift of a constant sign-bit will have exactly |
--- |
4077 |
// Similarly, a logical right-shift of a constant sign-bit will have exactly |
--- |
| 4078 |
// one bit set. |
--- |
4078 |
// one bit set. |
--- |
| 4079 |
if (Val.getOpcode() == ISD::SRL) { |
0 |
4079 |
if (Val.getOpcode() == ISD::SRL) { |
0 |
| 4080 |
auto *C = isConstOrConstSplat(Val.getOperand(0)); |
0 |
4080 |
auto *C = isConstOrConstSplat(Val.getOperand(0)); |
0 |
| 4081 |
if (C && C->getAPIntValue().isSignMask()) |
0 |
4081 |
if (C && C->getAPIntValue().isSignMask()) |
0 |
| 4082 |
return true; |
0 |
4082 |
return true; |
0 |
| 4083 |
} |
--- |
4083 |
} |
--- |
| 4084 |
|
--- |
4084 |
|
--- |
| 4085 |
// Are all operands of a build vector constant powers of two? |
--- |
4085 |
// Are all operands of a build vector constant powers of two? |
--- |
| 4086 |
if (Val.getOpcode() == ISD::BUILD_VECTOR) |
0 |
4086 |
if (Val.getOpcode() == ISD::BUILD_VECTOR) |
0 |
| 4087 |
if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) { |
0 |
4087 |
if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) { |
0 |
| 4088 |
if (ConstantSDNode *C = dyn_cast(E)) |
0 |
4088 |
if (ConstantSDNode *C = dyn_cast(E)) |
0 |
| 4089 |
return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2(); |
0 |
4089 |
return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2(); |
0 |
| 4090 |
return false; |
0 |
4090 |
return false; |
0 |
| 4091 |
})) |
--- |
4091 |
})) |
--- |
| 4092 |
return true; |
0 |
4092 |
return true; |
0 |
| 4093 |
|
--- |
4093 |
|
--- |
| 4094 |
// Is the operand of a splat vector a constant power of two? |
--- |
4094 |
// Is the operand of a splat vector a constant power of two? |
--- |
| 4095 |
if (Val.getOpcode() == ISD::SPLAT_VECTOR) |
0 |
4095 |
if (Val.getOpcode() == ISD::SPLAT_VECTOR) |
0 |
| 4096 |
if (ConstantSDNode *C = dyn_cast(Val->getOperand(0))) |
0 |
4096 |
if (ConstantSDNode *C = dyn_cast(Val->getOperand(0))) |
0 |
| 4097 |
if (C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2()) |
0 |
4097 |
if (C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2()) |
0 |
| 4098 |
return true; |
0 |
4098 |
return true; |
0 |
| 4099 |
|
--- |
4099 |
|
--- |
| 4100 |
// vscale(power-of-two) is a power-of-two for some targets |
--- |
4100 |
// vscale(power-of-two) is a power-of-two for some targets |
--- |
| 4101 |
if (Val.getOpcode() == ISD::VSCALE && |
0 |
4101 |
if (Val.getOpcode() == ISD::VSCALE && |
0 |
| 4102 |
getTargetLoweringInfo().isVScaleKnownToBeAPowerOfTwo() && |
0 |
4102 |
getTargetLoweringInfo().isVScaleKnownToBeAPowerOfTwo() && |
0 |
| 4103 |
isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1)) |
0 |
4103 |
isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1)) |
0 |
| 4104 |
return true; |
0 |
4104 |
return true; |
0 |
| 4105 |
|
--- |
4105 |
|
--- |
| 4106 |
// More could be done here, though the above checks are enough |
--- |
4106 |
// More could be done here, though the above checks are enough |
--- |
| 4107 |
// to handle some common cases. |
--- |
4107 |
// to handle some common cases. |
--- |
| 4108 |
return false; |
0 |
4108 |
return false; |
0 |
| 4109 |
} |
--- |
4109 |
} |
--- |
| 4110 |
|
--- |
4110 |
|
--- |
| 4111 |
unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const { |
0 |
4111 |
unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const { |
0 |
| 4112 |
EVT VT = Op.getValueType(); |
0 |
4112 |
EVT VT = Op.getValueType(); |
0 |
| 4113 |
|
--- |
4113 |
|
--- |
| 4114 |
// Since the number of lanes in a scalable vector is unknown at compile time, |
--- |
4114 |
// Since the number of lanes in a scalable vector is unknown at compile time, |
--- |
| 4115 |
// we track one bit which is implicitly broadcast to all lanes. This means |
--- |
4115 |
// we track one bit which is implicitly broadcast to all lanes. This means |
--- |
| 4116 |
// that all lanes in a scalable vector are considered demanded. |
--- |
4116 |
// that all lanes in a scalable vector are considered demanded. |
--- |
| 4117 |
APInt DemandedElts = VT.isFixedLengthVector() |
0 |
4117 |
APInt DemandedElts = VT.isFixedLengthVector() |
0 |
| 4118 |
? APInt::getAllOnes(VT.getVectorNumElements()) |
--- |
4118 |
? APInt::getAllOnes(VT.getVectorNumElements()) |
--- |
| 4119 |
: APInt(1, 1); |
0 |
4119 |
: APInt(1, 1); |
0 |
| 4120 |
return ComputeNumSignBits(Op, DemandedElts, Depth); |
0 |
4120 |
return ComputeNumSignBits(Op, DemandedElts, Depth); |
0 |
| 4121 |
} |
0 |
4121 |
} |
0 |
| 4122 |
|
--- |
4122 |
|
--- |
| 4123 |
unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, |
0 |
4123 |
unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, |
0 |
| 4124 |
unsigned Depth) const { |
--- |
4124 |
unsigned Depth) const { |
--- |
| 4125 |
EVT VT = Op.getValueType(); |
0 |
4125 |
EVT VT = Op.getValueType(); |
0 |
| 4126 |
assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!"); |
0 |
4126 |
assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!"); |
0 |
| 4127 |
unsigned VTBits = VT.getScalarSizeInBits(); |
0 |
4127 |
unsigned VTBits = VT.getScalarSizeInBits(); |
0 |
| 4128 |
unsigned NumElts = DemandedElts.getBitWidth(); |
0 |
4128 |
unsigned NumElts = DemandedElts.getBitWidth(); |
0 |
| 4129 |
unsigned Tmp, Tmp2; |
--- |
4129 |
unsigned Tmp, Tmp2; |
--- |
| 4130 |
unsigned FirstAnswer = 1; |
0 |
4130 |
unsigned FirstAnswer = 1; |
0 |
| 4131 |
|
--- |
4131 |
|
--- |
| 4132 |
if (auto *C = dyn_cast(Op)) { |
0 |
4132 |
if (auto *C = dyn_cast(Op)) { |
0 |
| 4133 |
const APInt &Val = C->getAPIntValue(); |
0 |
4133 |
const APInt &Val = C->getAPIntValue(); |
0 |
| 4134 |
return Val.getNumSignBits(); |
0 |
4134 |
return Val.getNumSignBits(); |
0 |
| 4135 |
} |
--- |
4135 |
} |
--- |
| 4136 |
|
--- |
4136 |
|
--- |
| 4137 |
if (Depth >= MaxRecursionDepth) |
0 |
4137 |
if (Depth >= MaxRecursionDepth) |
0 |
| 4138 |
return 1; // Limit search depth. |
0 |
4138 |
return 1; // Limit search depth. |
0 |
| 4139 |
|
--- |
4139 |
|
--- |
| 4140 |
if (!DemandedElts) |
0 |
4140 |
if (!DemandedElts) |
0 |
| 4141 |
return 1; // No demanded elts, better to assume we don't know anything. |
0 |
4141 |
return 1; // No demanded elts, better to assume we don't know anything. |
0 |
| 4142 |
|
--- |
4142 |
|
--- |
| 4143 |
unsigned Opcode = Op.getOpcode(); |
0 |
4143 |
unsigned Opcode = Op.getOpcode(); |
0 |
| 4144 |
switch (Opcode) { |
0 |
4144 |
switch (Opcode) { |
0 |
| 4145 |
default: break; |
0 |
4145 |
default: break; |
0 |
| 4146 |
case ISD::AssertSext: |
0 |
4146 |
case ISD::AssertSext: |
0 |
| 4147 |
Tmp = cast(Op.getOperand(1))->getVT().getSizeInBits(); |
0 |
4147 |
Tmp = cast(Op.getOperand(1))->getVT().getSizeInBits(); |
0 |
| 4148 |
return VTBits-Tmp+1; |
0 |
4148 |
return VTBits-Tmp+1; |
0 |
| 4149 |
case ISD::AssertZext: |
0 |
4149 |
case ISD::AssertZext: |
0 |
| 4150 |
Tmp = cast(Op.getOperand(1))->getVT().getSizeInBits(); |
0 |
4150 |
Tmp = cast(Op.getOperand(1))->getVT().getSizeInBits(); |
0 |
| 4151 |
return VTBits-Tmp; |
0 |
4151 |
return VTBits-Tmp; |
0 |
| 4152 |
case ISD::MERGE_VALUES: |
0 |
4152 |
case ISD::MERGE_VALUES: |
0 |
| 4153 |
return ComputeNumSignBits(Op.getOperand(Op.getResNo()), DemandedElts, |
0 |
4153 |
return ComputeNumSignBits(Op.getOperand(Op.getResNo()), DemandedElts, |
0 |
| 4154 |
Depth + 1); |
0 |
4154 |
Depth + 1); |
0 |
| 4155 |
case ISD::SPLAT_VECTOR: { |
0 |
4155 |
case ISD::SPLAT_VECTOR: { |
0 |
| 4156 |
// Check if the sign bits of source go down as far as the truncated value. |
--- |
4156 |
// Check if the sign bits of source go down as far as the truncated value. |
--- |
| 4157 |
unsigned NumSrcBits = Op.getOperand(0).getValueSizeInBits(); |
0 |
4157 |
unsigned NumSrcBits = Op.getOperand(0).getValueSizeInBits(); |
0 |
| 4158 |
unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1); |
0 |
4158 |
unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1); |
0 |
| 4159 |
if (NumSrcSignBits > (NumSrcBits - VTBits)) |
0 |
4159 |
if (NumSrcSignBits > (NumSrcBits - VTBits)) |
0 |
| 4160 |
return NumSrcSignBits - (NumSrcBits - VTBits); |
0 |
4160 |
return NumSrcSignBits - (NumSrcBits - VTBits); |
0 |
| 4161 |
break; |
0 |
4161 |
break; |
0 |
| 4162 |
} |
--- |
4162 |
} |
--- |
| 4163 |
case ISD::BUILD_VECTOR: |
0 |
4163 |
case ISD::BUILD_VECTOR: |
0 |
| 4164 |
assert(!VT.isScalableVector()); |
0 |
4164 |
assert(!VT.isScalableVector()); |
0 |
| 4165 |
Tmp = VTBits; |
0 |
4165 |
Tmp = VTBits; |
0 |
| 4166 |
for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) { |
0 |
4166 |
for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) { |
0 |
| 4167 |
if (!DemandedElts[i]) |
0 |
4167 |
if (!DemandedElts[i]) |
0 |
| 4168 |
continue; |
0 |
4168 |
continue; |
0 |
| 4169 |
|
--- |
4169 |
|
--- |
| 4170 |
SDValue SrcOp = Op.getOperand(i); |
0 |
4170 |
SDValue SrcOp = Op.getOperand(i); |
0 |
| 4171 |
// BUILD_VECTOR can implicitly truncate sources, we handle this specially |
--- |
4171 |
// BUILD_VECTOR can implicitly truncate sources, we handle this specially |
--- |
| 4172 |
// for constant nodes to ensure we only look at the sign bits. |
--- |
4172 |
// for constant nodes to ensure we only look at the sign bits. |
--- |
| 4173 |
if (ConstantSDNode *C = dyn_cast(SrcOp)) { |
0 |
4173 |
if (ConstantSDNode *C = dyn_cast(SrcOp)) { |
0 |
| 4174 |
APInt T = C->getAPIntValue().trunc(VTBits); |
0 |
4174 |
APInt T = C->getAPIntValue().trunc(VTBits); |
0 |
| 4175 |
Tmp2 = T.getNumSignBits(); |
0 |
4175 |
Tmp2 = T.getNumSignBits(); |
0 |
| 4176 |
} else { |
0 |
4176 |
} else { |
0 |
| 4177 |
Tmp2 = ComputeNumSignBits(SrcOp, Depth + 1); |
0 |
4177 |
Tmp2 = ComputeNumSignBits(SrcOp, Depth + 1); |
0 |
| 4178 |
|
--- |
4178 |
|
--- |
| 4179 |
if (SrcOp.getValueSizeInBits() != VTBits) { |
0 |
4179 |
if (SrcOp.getValueSizeInBits() != VTBits) { |
0 |
| 4180 |
assert(SrcOp.getValueSizeInBits() > VTBits && |
0 |
4180 |
assert(SrcOp.getValueSizeInBits() > VTBits && |
0 |
| 4181 |
"Expected BUILD_VECTOR implicit truncation"); |
--- |
4181 |
"Expected BUILD_VECTOR implicit truncation"); |
--- |
| 4182 |
unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits; |
0 |
4182 |
unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits; |
0 |
| 4183 |
Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1); |
0 |
4183 |
Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1); |
0 |
| 4184 |
} |
--- |
4184 |
} |
--- |
| 4185 |
} |
--- |
4185 |
} |
--- |
| 4186 |
Tmp = std::min(Tmp, Tmp2); |
0 |
4186 |
Tmp = std::min(Tmp, Tmp2); |
0 |
| 4187 |
} |
--- |
4187 |
} |
--- |
| 4188 |
return Tmp; |
0 |
4188 |
return Tmp; |
0 |
| 4189 |
|
--- |
4189 |
|
--- |
| 4190 |
case ISD::VECTOR_SHUFFLE: { |
0 |
4190 |
case ISD::VECTOR_SHUFFLE: { |
0 |
| 4191 |
// Collect the minimum number of sign bits that are shared by every vector |
--- |
4191 |
// Collect the minimum number of sign bits that are shared by every vector |
--- |
| 4192 |
// element referenced by the shuffle. |
--- |
4192 |
// element referenced by the shuffle. |
--- |
| 4193 |
APInt DemandedLHS, DemandedRHS; |
0 |
4193 |
APInt DemandedLHS, DemandedRHS; |
0 |
| 4194 |
const ShuffleVectorSDNode *SVN = cast(Op); |
0 |
4194 |
const ShuffleVectorSDNode *SVN = cast(Op); |
0 |
| 4195 |
assert(NumElts == SVN->getMask().size() && "Unexpected vector size"); |
0 |
4195 |
assert(NumElts == SVN->getMask().size() && "Unexpected vector size"); |
0 |
| 4196 |
if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts, |
0 |
4196 |
if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts, |
0 |
| 4197 |
DemandedLHS, DemandedRHS)) |
--- |
4197 |
DemandedLHS, DemandedRHS)) |
--- |
| 4198 |
return 1; |
0 |
4198 |
return 1; |
0 |
| 4199 |
|
--- |
4199 |
|
--- |
| 4200 |
Tmp = std::numeric_limits::max(); |
0 |
4200 |
Tmp = std::numeric_limits::max(); |
0 |
| 4201 |
if (!!DemandedLHS) |
0 |
4201 |
if (!!DemandedLHS) |
0 |
| 4202 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1); |
0 |
4202 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1); |
0 |
| 4203 |
if (!!DemandedRHS) { |
0 |
4203 |
if (!!DemandedRHS) { |
0 |
| 4204 |
Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1); |
0 |
4204 |
Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1); |
0 |
| 4205 |
Tmp = std::min(Tmp, Tmp2); |
0 |
4205 |
Tmp = std::min(Tmp, Tmp2); |
0 |
| 4206 |
} |
--- |
4206 |
} |
--- |
| 4207 |
// If we don't know anything, early out and try computeKnownBits fall-back. |
--- |
4207 |
// If we don't know anything, early out and try computeKnownBits fall-back. |
--- |
| 4208 |
if (Tmp == 1) |
0 |
4208 |
if (Tmp == 1) |
0 |
| 4209 |
break; |
0 |
4209 |
break; |
0 |
| 4210 |
assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); |
0 |
4210 |
assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); |
0 |
| 4211 |
return Tmp; |
0 |
4211 |
return Tmp; |
0 |
| 4212 |
} |
0 |
4212 |
} |
0 |
| 4213 |
|
--- |
4213 |
|
--- |
| 4214 |
case ISD::BITCAST: { |
0 |
4214 |
case ISD::BITCAST: { |
0 |
| 4215 |
if (VT.isScalableVector()) |
0 |
4215 |
if (VT.isScalableVector()) |
0 |
| 4216 |
break; |
0 |
4216 |
break; |
0 |
| 4217 |
SDValue N0 = Op.getOperand(0); |
0 |
4217 |
SDValue N0 = Op.getOperand(0); |
0 |
| 4218 |
EVT SrcVT = N0.getValueType(); |
0 |
4218 |
EVT SrcVT = N0.getValueType(); |
0 |
| 4219 |
unsigned SrcBits = SrcVT.getScalarSizeInBits(); |
0 |
4219 |
unsigned SrcBits = SrcVT.getScalarSizeInBits(); |
0 |
| 4220 |
|
--- |
4220 |
|
--- |
| 4221 |
// Ignore bitcasts from unsupported types.. |
--- |
4221 |
// Ignore bitcasts from unsupported types.. |
--- |
| 4222 |
if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint())) |
0 |
4222 |
if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint())) |
0 |
| 4223 |
break; |
0 |
4223 |
break; |
0 |
| 4224 |
|
--- |
4224 |
|
--- |
| 4225 |
// Fast handling of 'identity' bitcasts. |
--- |
4225 |
// Fast handling of 'identity' bitcasts. |
--- |
| 4226 |
if (VTBits == SrcBits) |
0 |
4226 |
if (VTBits == SrcBits) |
0 |
| 4227 |
return ComputeNumSignBits(N0, DemandedElts, Depth + 1); |
0 |
4227 |
return ComputeNumSignBits(N0, DemandedElts, Depth + 1); |
0 |
| 4228 |
|
--- |
4228 |
|
--- |
| 4229 |
bool IsLE = getDataLayout().isLittleEndian(); |
0 |
4229 |
bool IsLE = getDataLayout().isLittleEndian(); |
0 |
| 4230 |
|
--- |
4230 |
|
--- |
| 4231 |
// Bitcast 'large element' scalar/vector to 'small element' vector. |
--- |
4231 |
// Bitcast 'large element' scalar/vector to 'small element' vector. |
--- |
| 4232 |
if ((SrcBits % VTBits) == 0) { |
0 |
4232 |
if ((SrcBits % VTBits) == 0) { |
0 |
| 4233 |
assert(VT.isVector() && "Expected bitcast to vector"); |
0 |
4233 |
assert(VT.isVector() && "Expected bitcast to vector"); |
0 |
| 4234 |
|
--- |
4234 |
|
--- |
| 4235 |
unsigned Scale = SrcBits / VTBits; |
0 |
4235 |
unsigned Scale = SrcBits / VTBits; |
0 |
| 4236 |
APInt SrcDemandedElts = |
--- |
4236 |
APInt SrcDemandedElts = |
--- |
| 4237 |
APIntOps::ScaleBitMask(DemandedElts, NumElts / Scale); |
0 |
4237 |
APIntOps::ScaleBitMask(DemandedElts, NumElts / Scale); |
0 |
| 4238 |
|
--- |
4238 |
|
--- |
| 4239 |
// Fast case - sign splat can be simply split across the small elements. |
--- |
4239 |
// Fast case - sign splat can be simply split across the small elements. |
--- |
| 4240 |
Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1); |
0 |
4240 |
Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1); |
0 |
| 4241 |
if (Tmp == SrcBits) |
0 |
4241 |
if (Tmp == SrcBits) |
0 |
| 4242 |
return VTBits; |
0 |
4242 |
return VTBits; |
0 |
| 4243 |
|
--- |
4243 |
|
--- |
| 4244 |
// Slow case - determine how far the sign extends into each sub-element. |
--- |
4244 |
// Slow case - determine how far the sign extends into each sub-element. |
--- |
| 4245 |
Tmp2 = VTBits; |
0 |
4245 |
Tmp2 = VTBits; |
0 |
| 4246 |
for (unsigned i = 0; i != NumElts; ++i) |
0 |
4246 |
for (unsigned i = 0; i != NumElts; ++i) |
0 |
| 4247 |
if (DemandedElts[i]) { |
0 |
4247 |
if (DemandedElts[i]) { |
0 |
| 4248 |
unsigned SubOffset = i % Scale; |
0 |
4248 |
unsigned SubOffset = i % Scale; |
0 |
| 4249 |
SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset); |
0 |
4249 |
SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset); |
0 |
| 4250 |
SubOffset = SubOffset * VTBits; |
0 |
4250 |
SubOffset = SubOffset * VTBits; |
0 |
| 4251 |
if (Tmp <= SubOffset) |
0 |
4251 |
if (Tmp <= SubOffset) |
0 |
| 4252 |
return 1; |
0 |
4252 |
return 1; |
0 |
| 4253 |
Tmp2 = std::min(Tmp2, Tmp - SubOffset); |
0 |
4253 |
Tmp2 = std::min(Tmp2, Tmp - SubOffset); |
0 |
| 4254 |
} |
--- |
4254 |
} |
--- |
| 4255 |
return Tmp2; |
0 |
4255 |
return Tmp2; |
0 |
| 4256 |
} |
0 |
4256 |
} |
0 |
| 4257 |
break; |
0 |
4257 |
break; |
0 |
| 4258 |
} |
--- |
4258 |
} |
--- |
| 4259 |
|
--- |
4259 |
|
--- |
| 4260 |
case ISD::FP_TO_SINT_SAT: |
0 |
4260 |
case ISD::FP_TO_SINT_SAT: |
0 |
| 4261 |
// FP_TO_SINT_SAT produces a signed value that fits in the saturating VT. |
--- |
4261 |
// FP_TO_SINT_SAT produces a signed value that fits in the saturating VT. |
--- |
| 4262 |
Tmp = cast(Op.getOperand(1))->getVT().getScalarSizeInBits(); |
0 |
4262 |
Tmp = cast(Op.getOperand(1))->getVT().getScalarSizeInBits(); |
0 |
| 4263 |
return VTBits - Tmp + 1; |
0 |
4263 |
return VTBits - Tmp + 1; |
0 |
| 4264 |
case ISD::SIGN_EXTEND: |
0 |
4264 |
case ISD::SIGN_EXTEND: |
0 |
| 4265 |
Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits(); |
0 |
4265 |
Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits(); |
0 |
| 4266 |
return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp; |
0 |
4266 |
return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp; |
0 |
| 4267 |
case ISD::SIGN_EXTEND_INREG: |
0 |
4267 |
case ISD::SIGN_EXTEND_INREG: |
0 |
| 4268 |
// Max of the input and what this extends. |
--- |
4268 |
// Max of the input and what this extends. |
--- |
| 4269 |
Tmp = cast(Op.getOperand(1))->getVT().getScalarSizeInBits(); |
0 |
4269 |
Tmp = cast(Op.getOperand(1))->getVT().getScalarSizeInBits(); |
0 |
| 4270 |
Tmp = VTBits-Tmp+1; |
0 |
4270 |
Tmp = VTBits-Tmp+1; |
0 |
| 4271 |
Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1); |
0 |
4271 |
Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1); |
0 |
| 4272 |
return std::max(Tmp, Tmp2); |
0 |
4272 |
return std::max(Tmp, Tmp2); |
0 |
| 4273 |
case ISD::SIGN_EXTEND_VECTOR_INREG: { |
0 |
4273 |
case ISD::SIGN_EXTEND_VECTOR_INREG: { |
0 |
| 4274 |
if (VT.isScalableVector()) |
0 |
4274 |
if (VT.isScalableVector()) |
0 |
| 4275 |
break; |
0 |
4275 |
break; |
0 |
| 4276 |
SDValue Src = Op.getOperand(0); |
0 |
4276 |
SDValue Src = Op.getOperand(0); |
0 |
| 4277 |
EVT SrcVT = Src.getValueType(); |
0 |
4277 |
EVT SrcVT = Src.getValueType(); |
0 |
| 4278 |
APInt DemandedSrcElts = DemandedElts.zext(SrcVT.getVectorNumElements()); |
0 |
4278 |
APInt DemandedSrcElts = DemandedElts.zext(SrcVT.getVectorNumElements()); |
0 |
| 4279 |
Tmp = VTBits - SrcVT.getScalarSizeInBits(); |
0 |
4279 |
Tmp = VTBits - SrcVT.getScalarSizeInBits(); |
0 |
| 4280 |
return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp; |
0 |
4280 |
return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp; |
0 |
| 4281 |
} |
0 |
4281 |
} |
0 |
| 4282 |
case ISD::SRA: |
0 |
4282 |
case ISD::SRA: |
0 |
| 4283 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
4283 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 4284 |
// SRA X, C -> adds C sign bits. |
--- |
4284 |
// SRA X, C -> adds C sign bits. |
--- |
| 4285 |
if (const APInt *ShAmt = |
0 |
4285 |
if (const APInt *ShAmt = |
0 |
| 4286 |
getValidMinimumShiftAmountConstant(Op, DemandedElts)) |
0 |
4286 |
getValidMinimumShiftAmountConstant(Op, DemandedElts)) |
0 |
| 4287 |
Tmp = std::min(Tmp + ShAmt->getZExtValue(), VTBits); |
0 |
4287 |
Tmp = std::min(Tmp + ShAmt->getZExtValue(), VTBits); |
0 |
| 4288 |
return Tmp; |
0 |
4288 |
return Tmp; |
0 |
| 4289 |
case ISD::SHL: |
0 |
4289 |
case ISD::SHL: |
0 |
| 4290 |
if (const APInt *ShAmt = |
0 |
4290 |
if (const APInt *ShAmt = |
0 |
| 4291 |
getValidMaximumShiftAmountConstant(Op, DemandedElts)) { |
0 |
4291 |
getValidMaximumShiftAmountConstant(Op, DemandedElts)) { |
0 |
| 4292 |
// shl destroys sign bits, ensure it doesn't shift out all sign bits. |
--- |
4292 |
// shl destroys sign bits, ensure it doesn't shift out all sign bits. |
--- |
| 4293 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
4293 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 4294 |
if (ShAmt->ult(Tmp)) |
0 |
4294 |
if (ShAmt->ult(Tmp)) |
0 |
| 4295 |
return Tmp - ShAmt->getZExtValue(); |
0 |
4295 |
return Tmp - ShAmt->getZExtValue(); |
0 |
| 4296 |
} |
--- |
4296 |
} |
--- |
| 4297 |
break; |
0 |
4297 |
break; |
0 |
| 4298 |
case ISD::AND: |
0 |
4298 |
case ISD::AND: |
0 |
| 4299 |
case ISD::OR: |
--- |
4299 |
case ISD::OR: |
--- |
| 4300 |
case ISD::XOR: // NOT is handled here. |
--- |
4300 |
case ISD::XOR: // NOT is handled here. |
--- |
| 4301 |
// Logical binary ops preserve the number of sign bits at the worst. |
--- |
4301 |
// Logical binary ops preserve the number of sign bits at the worst. |
--- |
| 4302 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1); |
0 |
4302 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1); |
0 |
| 4303 |
if (Tmp != 1) { |
0 |
4303 |
if (Tmp != 1) { |
0 |
| 4304 |
Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1); |
0 |
4304 |
Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1); |
0 |
| 4305 |
FirstAnswer = std::min(Tmp, Tmp2); |
0 |
4305 |
FirstAnswer = std::min(Tmp, Tmp2); |
0 |
| 4306 |
// We computed what we know about the sign bits as our first |
--- |
4306 |
// We computed what we know about the sign bits as our first |
--- |
| 4307 |
// answer. Now proceed to the generic code that uses |
--- |
4307 |
// answer. Now proceed to the generic code that uses |
--- |
| 4308 |
// computeKnownBits, and pick whichever answer is better. |
--- |
4308 |
// computeKnownBits, and pick whichever answer is better. |
--- |
| 4309 |
} |
--- |
4309 |
} |
--- |
| 4310 |
break; |
0 |
4310 |
break; |
0 |
| 4311 |
|
--- |
4311 |
|
--- |
| 4312 |
case ISD::SELECT: |
0 |
4312 |
case ISD::SELECT: |
0 |
| 4313 |
case ISD::VSELECT: |
--- |
4313 |
case ISD::VSELECT: |
--- |
| 4314 |
Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1); |
0 |
4314 |
Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1); |
0 |
| 4315 |
if (Tmp == 1) return 1; // Early out. |
0 |
4315 |
if (Tmp == 1) return 1; // Early out. |
0 |
| 4316 |
Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1); |
0 |
4316 |
Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1); |
0 |
| 4317 |
return std::min(Tmp, Tmp2); |
0 |
4317 |
return std::min(Tmp, Tmp2); |
0 |
| 4318 |
case ISD::SELECT_CC: |
0 |
4318 |
case ISD::SELECT_CC: |
0 |
| 4319 |
Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1); |
0 |
4319 |
Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1); |
0 |
| 4320 |
if (Tmp == 1) return 1; // Early out. |
0 |
4320 |
if (Tmp == 1) return 1; // Early out. |
0 |
| 4321 |
Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1); |
0 |
4321 |
Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1); |
0 |
| 4322 |
return std::min(Tmp, Tmp2); |
0 |
4322 |
return std::min(Tmp, Tmp2); |
0 |
| 4323 |
|
--- |
4323 |
|
--- |
| 4324 |
case ISD::SMIN: |
0 |
4324 |
case ISD::SMIN: |
0 |
| 4325 |
case ISD::SMAX: { |
--- |
4325 |
case ISD::SMAX: { |
--- |
| 4326 |
// If we have a clamp pattern, we know that the number of sign bits will be |
--- |
4326 |
// If we have a clamp pattern, we know that the number of sign bits will be |
--- |
| 4327 |
// the minimum of the clamp min/max range. |
--- |
4327 |
// the minimum of the clamp min/max range. |
--- |
| 4328 |
bool IsMax = (Opcode == ISD::SMAX); |
0 |
4328 |
bool IsMax = (Opcode == ISD::SMAX); |
0 |
| 4329 |
ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr; |
0 |
4329 |
ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr; |
0 |
| 4330 |
if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts))) |
0 |
4330 |
if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts))) |
0 |
| 4331 |
if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) |
0 |
4331 |
if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) |
0 |
| 4332 |
CstHigh = |
0 |
4332 |
CstHigh = |
0 |
| 4333 |
isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts); |
0 |
4333 |
isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts); |
0 |
| 4334 |
if (CstLow && CstHigh) { |
0 |
4334 |
if (CstLow && CstHigh) { |
0 |
| 4335 |
if (!IsMax) |
0 |
4335 |
if (!IsMax) |
0 |
| 4336 |
std::swap(CstLow, CstHigh); |
0 |
4336 |
std::swap(CstLow, CstHigh); |
0 |
| 4337 |
if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) { |
0 |
4337 |
if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) { |
0 |
| 4338 |
Tmp = CstLow->getAPIntValue().getNumSignBits(); |
0 |
4338 |
Tmp = CstLow->getAPIntValue().getNumSignBits(); |
0 |
| 4339 |
Tmp2 = CstHigh->getAPIntValue().getNumSignBits(); |
0 |
4339 |
Tmp2 = CstHigh->getAPIntValue().getNumSignBits(); |
0 |
| 4340 |
return std::min(Tmp, Tmp2); |
0 |
4340 |
return std::min(Tmp, Tmp2); |
0 |
| 4341 |
} |
--- |
4341 |
} |
--- |
| 4342 |
} |
--- |
4342 |
} |
--- |
| 4343 |
|
--- |
4343 |
|
--- |
| 4344 |
// Fallback - just get the minimum number of sign bits of the operands. |
--- |
4344 |
// Fallback - just get the minimum number of sign bits of the operands. |
--- |
| 4345 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
4345 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 4346 |
if (Tmp == 1) |
0 |
4346 |
if (Tmp == 1) |
0 |
| 4347 |
return 1; // Early out. |
0 |
4347 |
return 1; // Early out. |
0 |
| 4348 |
Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
4348 |
Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 4349 |
return std::min(Tmp, Tmp2); |
0 |
4349 |
return std::min(Tmp, Tmp2); |
0 |
| 4350 |
} |
--- |
4350 |
} |
--- |
| 4351 |
case ISD::UMIN: |
0 |
4351 |
case ISD::UMIN: |
0 |
| 4352 |
case ISD::UMAX: |
--- |
4352 |
case ISD::UMAX: |
--- |
| 4353 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
4353 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 4354 |
if (Tmp == 1) |
0 |
4354 |
if (Tmp == 1) |
0 |
| 4355 |
return 1; // Early out. |
0 |
4355 |
return 1; // Early out. |
0 |
| 4356 |
Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
4356 |
Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 4357 |
return std::min(Tmp, Tmp2); |
0 |
4357 |
return std::min(Tmp, Tmp2); |
0 |
| 4358 |
case ISD::SADDO: |
0 |
4358 |
case ISD::SADDO: |
0 |
| 4359 |
case ISD::UADDO: |
--- |
4359 |
case ISD::UADDO: |
--- |
| 4360 |
case ISD::SADDO_CARRY: |
--- |
4360 |
case ISD::SADDO_CARRY: |
--- |
| 4361 |
case ISD::UADDO_CARRY: |
--- |
4361 |
case ISD::UADDO_CARRY: |
--- |
| 4362 |
case ISD::SSUBO: |
--- |
4362 |
case ISD::SSUBO: |
--- |
| 4363 |
case ISD::USUBO: |
--- |
4363 |
case ISD::USUBO: |
--- |
| 4364 |
case ISD::SSUBO_CARRY: |
--- |
4364 |
case ISD::SSUBO_CARRY: |
--- |
| 4365 |
case ISD::USUBO_CARRY: |
--- |
4365 |
case ISD::USUBO_CARRY: |
--- |
| 4366 |
case ISD::SMULO: |
--- |
4366 |
case ISD::SMULO: |
--- |
| 4367 |
case ISD::UMULO: |
--- |
4367 |
case ISD::UMULO: |
--- |
| 4368 |
if (Op.getResNo() != 1) |
0 |
4368 |
if (Op.getResNo() != 1) |
0 |
| 4369 |
break; |
0 |
4369 |
break; |
0 |
| 4370 |
// The boolean result conforms to getBooleanContents. Fall through. |
--- |
4370 |
// The boolean result conforms to getBooleanContents. Fall through. |
--- |
| 4371 |
// If setcc returns 0/-1, all bits are sign bits. |
--- |
4371 |
// If setcc returns 0/-1, all bits are sign bits. |
--- |
| 4372 |
// We know that we have an integer-based boolean since these operations |
--- |
4372 |
// We know that we have an integer-based boolean since these operations |
--- |
| 4373 |
// are only available for integer. |
--- |
4373 |
// are only available for integer. |
--- |
| 4374 |
if (TLI->getBooleanContents(VT.isVector(), false) == |
0 |
4374 |
if (TLI->getBooleanContents(VT.isVector(), false) == |
0 |
| 4375 |
TargetLowering::ZeroOrNegativeOneBooleanContent) |
--- |
4375 |
TargetLowering::ZeroOrNegativeOneBooleanContent) |
--- |
| 4376 |
return VTBits; |
0 |
4376 |
return VTBits; |
0 |
| 4377 |
break; |
0 |
4377 |
break; |
0 |
| 4378 |
case ISD::SETCC: |
0 |
4378 |
case ISD::SETCC: |
0 |
| 4379 |
case ISD::SETCCCARRY: |
--- |
4379 |
case ISD::SETCCCARRY: |
--- |
| 4380 |
case ISD::STRICT_FSETCC: |
--- |
4380 |
case ISD::STRICT_FSETCC: |
--- |
| 4381 |
case ISD::STRICT_FSETCCS: { |
--- |
4381 |
case ISD::STRICT_FSETCCS: { |
--- |
| 4382 |
unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0; |
0 |
4382 |
unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0; |
0 |
| 4383 |
// If setcc returns 0/-1, all bits are sign bits. |
--- |
4383 |
// If setcc returns 0/-1, all bits are sign bits. |
--- |
| 4384 |
if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) == |
0 |
4384 |
if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) == |
0 |
| 4385 |
TargetLowering::ZeroOrNegativeOneBooleanContent) |
--- |
4385 |
TargetLowering::ZeroOrNegativeOneBooleanContent) |
--- |
| 4386 |
return VTBits; |
0 |
4386 |
return VTBits; |
0 |
| 4387 |
break; |
0 |
4387 |
break; |
0 |
| 4388 |
} |
--- |
4388 |
} |
--- |
| 4389 |
case ISD::ROTL: |
0 |
4389 |
case ISD::ROTL: |
0 |
| 4390 |
case ISD::ROTR: |
--- |
4390 |
case ISD::ROTR: |
--- |
| 4391 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
4391 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 4392 |
|
--- |
4392 |
|
--- |
| 4393 |
// If we're rotating an 0/-1 value, then it stays an 0/-1 value. |
--- |
4393 |
// If we're rotating an 0/-1 value, then it stays an 0/-1 value. |
--- |
| 4394 |
if (Tmp == VTBits) |
0 |
4394 |
if (Tmp == VTBits) |
0 |
| 4395 |
return VTBits; |
0 |
4395 |
return VTBits; |
0 |
| 4396 |
|
--- |
4396 |
|
--- |
| 4397 |
if (ConstantSDNode *C = |
0 |
4397 |
if (ConstantSDNode *C = |
0 |
| 4398 |
isConstOrConstSplat(Op.getOperand(1), DemandedElts)) { |
0 |
4398 |
isConstOrConstSplat(Op.getOperand(1), DemandedElts)) { |
0 |
| 4399 |
unsigned RotAmt = C->getAPIntValue().urem(VTBits); |
0 |
4399 |
unsigned RotAmt = C->getAPIntValue().urem(VTBits); |
0 |
| 4400 |
|
--- |
4400 |
|
--- |
| 4401 |
// Handle rotate right by N like a rotate left by 32-N. |
--- |
4401 |
// Handle rotate right by N like a rotate left by 32-N. |
--- |
| 4402 |
if (Opcode == ISD::ROTR) |
0 |
4402 |
if (Opcode == ISD::ROTR) |
0 |
| 4403 |
RotAmt = (VTBits - RotAmt) % VTBits; |
0 |
4403 |
RotAmt = (VTBits - RotAmt) % VTBits; |
0 |
| 4404 |
|
--- |
4404 |
|
--- |
| 4405 |
// If we aren't rotating out all of the known-in sign bits, return the |
--- |
4405 |
// If we aren't rotating out all of the known-in sign bits, return the |
--- |
| 4406 |
// number that are left. This handles rotl(sext(x), 1) for example. |
--- |
4406 |
// number that are left. This handles rotl(sext(x), 1) for example. |
--- |
| 4407 |
if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt); |
0 |
4407 |
if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt); |
0 |
| 4408 |
} |
--- |
4408 |
} |
--- |
| 4409 |
break; |
0 |
4409 |
break; |
0 |
| 4410 |
case ISD::ADD: |
0 |
4410 |
case ISD::ADD: |
0 |
| 4411 |
case ISD::ADDC: |
--- |
4411 |
case ISD::ADDC: |
--- |
| 4412 |
// Add can have at most one carry bit. Thus we know that the output |
--- |
4412 |
// Add can have at most one carry bit. Thus we know that the output |
--- |
| 4413 |
// is, at worst, one more bit than the inputs. |
--- |
4413 |
// is, at worst, one more bit than the inputs. |
--- |
| 4414 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
4414 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 4415 |
if (Tmp == 1) return 1; // Early out. |
0 |
4415 |
if (Tmp == 1) return 1; // Early out. |
0 |
| 4416 |
|
--- |
4416 |
|
--- |
| 4417 |
// Special case decrementing a value (ADD X, -1): |
--- |
4417 |
// Special case decrementing a value (ADD X, -1): |
--- |
| 4418 |
if (ConstantSDNode *CRHS = |
0 |
4418 |
if (ConstantSDNode *CRHS = |
0 |
| 4419 |
isConstOrConstSplat(Op.getOperand(1), DemandedElts)) |
0 |
4419 |
isConstOrConstSplat(Op.getOperand(1), DemandedElts)) |
0 |
| 4420 |
if (CRHS->isAllOnes()) { |
0 |
4420 |
if (CRHS->isAllOnes()) { |
0 |
| 4421 |
KnownBits Known = |
--- |
4421 |
KnownBits Known = |
--- |
| 4422 |
computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
4422 |
computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 4423 |
|
--- |
4423 |
|
--- |
| 4424 |
// If the input is known to be 0 or 1, the output is 0/-1, which is all |
--- |
4424 |
// If the input is known to be 0 or 1, the output is 0/-1, which is all |
--- |
| 4425 |
// sign bits set. |
--- |
4425 |
// sign bits set. |
--- |
| 4426 |
if ((Known.Zero | 1).isAllOnes()) |
0 |
4426 |
if ((Known.Zero | 1).isAllOnes()) |
0 |
| 4427 |
return VTBits; |
0 |
4427 |
return VTBits; |
0 |
| 4428 |
|
--- |
4428 |
|
--- |
| 4429 |
// If we are subtracting one from a positive number, there is no carry |
--- |
4429 |
// If we are subtracting one from a positive number, there is no carry |
--- |
| 4430 |
// out of the result. |
--- |
4430 |
// out of the result. |
--- |
| 4431 |
if (Known.isNonNegative()) |
0 |
4431 |
if (Known.isNonNegative()) |
0 |
| 4432 |
return Tmp; |
0 |
4432 |
return Tmp; |
0 |
| 4433 |
} |
0 |
4433 |
} |
0 |
| 4434 |
|
--- |
4434 |
|
--- |
| 4435 |
Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
4435 |
Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 4436 |
if (Tmp2 == 1) return 1; // Early out. |
0 |
4436 |
if (Tmp2 == 1) return 1; // Early out. |
0 |
| 4437 |
return std::min(Tmp, Tmp2) - 1; |
0 |
4437 |
return std::min(Tmp, Tmp2) - 1; |
0 |
| 4438 |
case ISD::SUB: |
0 |
4438 |
case ISD::SUB: |
0 |
| 4439 |
Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
4439 |
Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 4440 |
if (Tmp2 == 1) return 1; // Early out. |
0 |
4440 |
if (Tmp2 == 1) return 1; // Early out. |
0 |
| 4441 |
|
--- |
4441 |
|
--- |
| 4442 |
// Handle NEG. |
--- |
4442 |
// Handle NEG. |
--- |
| 4443 |
if (ConstantSDNode *CLHS = |
0 |
4443 |
if (ConstantSDNode *CLHS = |
0 |
| 4444 |
isConstOrConstSplat(Op.getOperand(0), DemandedElts)) |
0 |
4444 |
isConstOrConstSplat(Op.getOperand(0), DemandedElts)) |
0 |
| 4445 |
if (CLHS->isZero()) { |
0 |
4445 |
if (CLHS->isZero()) { |
0 |
| 4446 |
KnownBits Known = |
--- |
4446 |
KnownBits Known = |
--- |
| 4447 |
computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
4447 |
computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); |
0 |
| 4448 |
// If the input is known to be 0 or 1, the output is 0/-1, which is all |
--- |
4448 |
// If the input is known to be 0 or 1, the output is 0/-1, which is all |
--- |
| 4449 |
// sign bits set. |
--- |
4449 |
// sign bits set. |
--- |
| 4450 |
if ((Known.Zero | 1).isAllOnes()) |
0 |
4450 |
if ((Known.Zero | 1).isAllOnes()) |
0 |
| 4451 |
return VTBits; |
0 |
4451 |
return VTBits; |
0 |
| 4452 |
|
--- |
4452 |
|
--- |
| 4453 |
// If the input is known to be positive (the sign bit is known clear), |
--- |
4453 |
// If the input is known to be positive (the sign bit is known clear), |
--- |
| 4454 |
// the output of the NEG has the same number of sign bits as the input. |
--- |
4454 |
// the output of the NEG has the same number of sign bits as the input. |
--- |
| 4455 |
if (Known.isNonNegative()) |
0 |
4455 |
if (Known.isNonNegative()) |
0 |
| 4456 |
return Tmp2; |
0 |
4456 |
return Tmp2; |
0 |
| 4457 |
|
--- |
4457 |
|
--- |
| 4458 |
// Otherwise, we treat this like a SUB. |
--- |
4458 |
// Otherwise, we treat this like a SUB. |
--- |
| 4459 |
} |
0 |
4459 |
} |
0 |
| 4460 |
|
--- |
4460 |
|
--- |
| 4461 |
// Sub can have at most one carry bit. Thus we know that the output |
--- |
4461 |
// Sub can have at most one carry bit. Thus we know that the output |
--- |
| 4462 |
// is, at worst, one more bit than the inputs. |
--- |
4462 |
// is, at worst, one more bit than the inputs. |
--- |
| 4463 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
4463 |
Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 4464 |
if (Tmp == 1) return 1; // Early out. |
0 |
4464 |
if (Tmp == 1) return 1; // Early out. |
0 |
| 4465 |
return std::min(Tmp, Tmp2) - 1; |
0 |
4465 |
return std::min(Tmp, Tmp2) - 1; |
0 |
| 4466 |
case ISD::MUL: { |
0 |
4466 |
case ISD::MUL: { |
0 |
| 4467 |
// The output of the Mul can be at most twice the valid bits in the inputs. |
--- |
4467 |
// The output of the Mul can be at most twice the valid bits in the inputs. |
--- |
| 4468 |
unsigned SignBitsOp0 = ComputeNumSignBits(Op.getOperand(0), Depth + 1); |
0 |
4468 |
unsigned SignBitsOp0 = ComputeNumSignBits(Op.getOperand(0), Depth + 1); |
0 |
| 4469 |
if (SignBitsOp0 == 1) |
0 |
4469 |
if (SignBitsOp0 == 1) |
0 |
| 4470 |
break; |
0 |
4470 |
break; |
0 |
| 4471 |
unsigned SignBitsOp1 = ComputeNumSignBits(Op.getOperand(1), Depth + 1); |
0 |
4471 |
unsigned SignBitsOp1 = ComputeNumSignBits(Op.getOperand(1), Depth + 1); |
0 |
| 4472 |
if (SignBitsOp1 == 1) |
0 |
4472 |
if (SignBitsOp1 == 1) |
0 |
| 4473 |
break; |
0 |
4473 |
break; |
0 |
| 4474 |
unsigned OutValidBits = |
0 |
4474 |
unsigned OutValidBits = |
0 |
| 4475 |
(VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1); |
0 |
4475 |
(VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1); |
0 |
| 4476 |
return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1; |
0 |
4476 |
return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1; |
0 |
| 4477 |
} |
--- |
4477 |
} |
--- |
| 4478 |
case ISD::SREM: |
0 |
4478 |
case ISD::SREM: |
0 |
| 4479 |
// The sign bit is the LHS's sign bit, except when the result of the |
--- |
4479 |
// The sign bit is the LHS's sign bit, except when the result of the |
--- |
| 4480 |
// remainder is zero. The magnitude of the result should be less than or |
--- |
4480 |
// remainder is zero. The magnitude of the result should be less than or |
--- |
| 4481 |
// equal to the magnitude of the LHS. Therefore, the result should have |
--- |
4481 |
// equal to the magnitude of the LHS. Therefore, the result should have |
--- |
| 4482 |
// at least as many sign bits as the left hand side. |
--- |
4482 |
// at least as many sign bits as the left hand side. |
--- |
| 4483 |
return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
4483 |
return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); |
0 |
| 4484 |
case ISD::TRUNCATE: { |
0 |
4484 |
case ISD::TRUNCATE: { |
0 |
| 4485 |
// Check if the sign bits of source go down as far as the truncated value. |
--- |
4485 |
// Check if the sign bits of source go down as far as the truncated value. |
--- |
| 4486 |
unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits(); |
0 |
4486 |
unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits(); |
0 |
| 4487 |
unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1); |
0 |
4487 |
unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1); |
0 |
| 4488 |
if (NumSrcSignBits > (NumSrcBits - VTBits)) |
0 |
4488 |
if (NumSrcSignBits > (NumSrcBits - VTBits)) |
0 |
| 4489 |
return NumSrcSignBits - (NumSrcBits - VTBits); |
0 |
4489 |
return NumSrcSignBits - (NumSrcBits - VTBits); |
0 |
| 4490 |
break; |
0 |
4490 |
break; |
0 |
| 4491 |
} |
--- |
4491 |
} |
--- |
| 4492 |
case ISD::EXTRACT_ELEMENT: { |
0 |
4492 |
case ISD::EXTRACT_ELEMENT: { |
0 |
| 4493 |
if (VT.isScalableVector()) |
0 |
4493 |
if (VT.isScalableVector()) |
0 |
| 4494 |
break; |
0 |
4494 |
break; |
0 |
| 4495 |
const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1); |
0 |
4495 |
const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1); |
0 |
| 4496 |
const int BitWidth = Op.getValueSizeInBits(); |
0 |
4496 |
const int BitWidth = Op.getValueSizeInBits(); |
0 |
| 4497 |
const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth; |
0 |
4497 |
const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth; |
0 |
| 4498 |
|
--- |
4498 |
|
--- |
| 4499 |
// Get reverse index (starting from 1), Op1 value indexes elements from |
--- |
4499 |
// Get reverse index (starting from 1), Op1 value indexes elements from |
--- |
| 4500 |
// little end. Sign starts at big end. |
--- |
4500 |
// little end. Sign starts at big end. |
--- |
| 4501 |
const int rIndex = Items - 1 - Op.getConstantOperandVal(1); |
0 |
4501 |
const int rIndex = Items - 1 - Op.getConstantOperandVal(1); |
0 |
| 4502 |
|
--- |
4502 |
|
--- |
| 4503 |
// If the sign portion ends in our element the subtraction gives correct |
--- |
4503 |
// If the sign portion ends in our element the subtraction gives correct |
--- |
| 4504 |
// result. Otherwise it gives either negative or > bitwidth result |
--- |
4504 |
// result. Otherwise it gives either negative or > bitwidth result |
--- |
| 4505 |
return std::clamp(KnownSign - rIndex * BitWidth, 0, BitWidth); |
0 |
4505 |
return std::clamp(KnownSign - rIndex * BitWidth, 0, BitWidth); |
0 |
| 4506 |
} |
--- |
4506 |
} |
--- |
| 4507 |
case ISD::INSERT_VECTOR_ELT: { |
0 |
4507 |
case ISD::INSERT_VECTOR_ELT: { |
0 |
| 4508 |
if (VT.isScalableVector()) |
0 |
4508 |
if (VT.isScalableVector()) |
0 |
| 4509 |
break; |
0 |
4509 |
break; |
0 |
| 4510 |
// If we know the element index, split the demand between the |
--- |
4510 |
// If we know the element index, split the demand between the |
--- |
| 4511 |
// source vector and the inserted element, otherwise assume we need |
--- |
4511 |
// source vector and the inserted element, otherwise assume we need |
--- |
| 4512 |
// the original demanded vector elements and the value. |
--- |
4512 |
// the original demanded vector elements and the value. |
--- |
| 4513 |
SDValue InVec = Op.getOperand(0); |
0 |
4513 |
SDValue InVec = Op.getOperand(0); |
0 |
| 4514 |
SDValue InVal = Op.getOperand(1); |
0 |
4514 |
SDValue InVal = Op.getOperand(1); |
0 |
| 4515 |
SDValue EltNo = Op.getOperand(2); |
0 |
4515 |
SDValue EltNo = Op.getOperand(2); |
0 |
| 4516 |
bool DemandedVal = true; |
0 |
4516 |
bool DemandedVal = true; |
0 |
| 4517 |
APInt DemandedVecElts = DemandedElts; |
0 |
4517 |
APInt DemandedVecElts = DemandedElts; |
0 |
| 4518 |
auto *CEltNo = dyn_cast(EltNo); |
0 |
4518 |
auto *CEltNo = dyn_cast(EltNo); |
0 |
| 4519 |
if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) { |
0 |
4519 |
if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) { |
0 |
| 4520 |
unsigned EltIdx = CEltNo->getZExtValue(); |
0 |
4520 |
unsigned EltIdx = CEltNo->getZExtValue(); |
0 |
| 4521 |
DemandedVal = !!DemandedElts[EltIdx]; |
0 |
4521 |
DemandedVal = !!DemandedElts[EltIdx]; |
0 |
| 4522 |
DemandedVecElts.clearBit(EltIdx); |
0 |
4522 |
DemandedVecElts.clearBit(EltIdx); |
0 |
| 4523 |
} |
--- |
4523 |
} |
--- |
| 4524 |
Tmp = std::numeric_limits::max(); |
0 |
4524 |
Tmp = std::numeric_limits::max(); |
0 |
| 4525 |
if (DemandedVal) { |
0 |
4525 |
if (DemandedVal) { |
0 |
| 4526 |
// TODO - handle implicit truncation of inserted elements. |
--- |
4526 |
// TODO - handle implicit truncation of inserted elements. |
--- |
| 4527 |
if (InVal.getScalarValueSizeInBits() != VTBits) |
0 |
4527 |
if (InVal.getScalarValueSizeInBits() != VTBits) |
0 |
| 4528 |
break; |
0 |
4528 |
break; |
0 |
| 4529 |
Tmp2 = ComputeNumSignBits(InVal, Depth + 1); |
0 |
4529 |
Tmp2 = ComputeNumSignBits(InVal, Depth + 1); |
0 |
| 4530 |
Tmp = std::min(Tmp, Tmp2); |
0 |
4530 |
Tmp = std::min(Tmp, Tmp2); |
0 |
| 4531 |
} |
--- |
4531 |
} |
--- |
| 4532 |
if (!!DemandedVecElts) { |
0 |
4532 |
if (!!DemandedVecElts) { |
0 |
| 4533 |
Tmp2 = ComputeNumSignBits(InVec, DemandedVecElts, Depth + 1); |
0 |
4533 |
Tmp2 = ComputeNumSignBits(InVec, DemandedVecElts, Depth + 1); |
0 |
| 4534 |
Tmp = std::min(Tmp, Tmp2); |
0 |
4534 |
Tmp = std::min(Tmp, Tmp2); |
0 |
| 4535 |
} |
--- |
4535 |
} |
--- |
| 4536 |
assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); |
0 |
4536 |
assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); |
0 |
| 4537 |
return Tmp; |
0 |
4537 |
return Tmp; |
0 |
| 4538 |
} |
0 |
4538 |
} |
0 |
| 4539 |
case ISD::EXTRACT_VECTOR_ELT: { |
0 |
4539 |
case ISD::EXTRACT_VECTOR_ELT: { |
0 |
| 4540 |
assert(!VT.isScalableVector()); |
0 |
4540 |
assert(!VT.isScalableVector()); |
0 |
| 4541 |
SDValue InVec = Op.getOperand(0); |
0 |
4541 |
SDValue InVec = Op.getOperand(0); |
0 |
| 4542 |
SDValue EltNo = Op.getOperand(1); |
0 |
4542 |
SDValue EltNo = Op.getOperand(1); |
0 |
| 4543 |
EVT VecVT = InVec.getValueType(); |
0 |
4543 |
EVT VecVT = InVec.getValueType(); |
0 |
| 4544 |
// ComputeNumSignBits not yet implemented for scalable vectors. |
--- |
4544 |
// ComputeNumSignBits not yet implemented for scalable vectors. |
--- |
| 4545 |
if (VecVT.isScalableVector()) |
0 |
4545 |
if (VecVT.isScalableVector()) |
0 |
| 4546 |
break; |
0 |
4546 |
break; |
0 |
| 4547 |
const unsigned BitWidth = Op.getValueSizeInBits(); |
0 |
4547 |
const unsigned BitWidth = Op.getValueSizeInBits(); |
0 |
| 4548 |
const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits(); |
0 |
4548 |
const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits(); |
0 |
| 4549 |
const unsigned NumSrcElts = VecVT.getVectorNumElements(); |
0 |
4549 |
const unsigned NumSrcElts = VecVT.getVectorNumElements(); |
0 |
| 4550 |
|
--- |
4550 |
|
--- |
| 4551 |
// If BitWidth > EltBitWidth the value is anyext:ed, and we do not know |
--- |
4551 |
// If BitWidth > EltBitWidth the value is anyext:ed, and we do not know |
--- |
| 4552 |
// anything about sign bits. But if the sizes match we can derive knowledge |
--- |
4552 |
// anything about sign bits. But if the sizes match we can derive knowledge |
--- |
| 4553 |
// about sign bits from the vector operand. |
--- |
4553 |
// about sign bits from the vector operand. |
--- |
| 4554 |
if (BitWidth != EltBitWidth) |
0 |
4554 |
if (BitWidth != EltBitWidth) |
0 |
| 4555 |
break; |
0 |
4555 |
break; |
0 |
| 4556 |
|
--- |
4556 |
|
--- |
| 4557 |
// If we know the element index, just demand that vector element, else for |
--- |
4557 |
// If we know the element index, just demand that vector element, else for |
--- |
| 4558 |
// an unknown element index, ignore DemandedElts and demand them all. |
--- |
4558 |
// an unknown element index, ignore DemandedElts and demand them all. |
--- |
| 4559 |
APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts); |
0 |
4559 |
APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts); |
0 |
| 4560 |
auto *ConstEltNo = dyn_cast(EltNo); |
0 |
4560 |
auto *ConstEltNo = dyn_cast(EltNo); |
0 |
| 4561 |
if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts)) |
0 |
4561 |
if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts)) |
0 |
| 4562 |
DemandedSrcElts = |
--- |
4562 |
DemandedSrcElts = |
--- |
| 4563 |
APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue()); |
0 |
4563 |
APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue()); |
0 |
| 4564 |
|
--- |
4564 |
|
--- |
| 4565 |
return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1); |
0 |
4565 |
return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1); |
0 |
| 4566 |
} |
0 |
4566 |
} |
0 |
| 4567 |
case ISD::EXTRACT_SUBVECTOR: { |
0 |
4567 |
case ISD::EXTRACT_SUBVECTOR: { |
0 |
| 4568 |
// Offset the demanded elts by the subvector index. |
--- |
4568 |
// Offset the demanded elts by the subvector index. |
--- |
| 4569 |
SDValue Src = Op.getOperand(0); |
0 |
4569 |
SDValue Src = Op.getOperand(0); |
0 |
| 4570 |
// Bail until we can represent demanded elements for scalable vectors. |
--- |
4570 |
// Bail until we can represent demanded elements for scalable vectors. |
--- |
| 4571 |
if (Src.getValueType().isScalableVector()) |
0 |
4571 |
if (Src.getValueType().isScalableVector()) |
0 |
| 4572 |
break; |
0 |
4572 |
break; |
0 |
| 4573 |
uint64_t Idx = Op.getConstantOperandVal(1); |
0 |
4573 |
uint64_t Idx = Op.getConstantOperandVal(1); |
0 |
| 4574 |
unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); |
0 |
4574 |
unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); |
0 |
| 4575 |
APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx); |
0 |
4575 |
APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx); |
0 |
| 4576 |
return ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1); |
0 |
4576 |
return ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1); |
0 |
| 4577 |
} |
0 |
4577 |
} |
0 |
| 4578 |
case ISD::CONCAT_VECTORS: { |
0 |
4578 |
case ISD::CONCAT_VECTORS: { |
0 |
| 4579 |
if (VT.isScalableVector()) |
0 |
4579 |
if (VT.isScalableVector()) |
0 |
| 4580 |
break; |
0 |
4580 |
break; |
0 |
| 4581 |
// Determine the minimum number of sign bits across all demanded |
--- |
4581 |
// Determine the minimum number of sign bits across all demanded |
--- |
| 4582 |
// elts of the input vectors. Early out if the result is already 1. |
--- |
4582 |
// elts of the input vectors. Early out if the result is already 1. |
--- |
| 4583 |
Tmp = std::numeric_limits::max(); |
0 |
4583 |
Tmp = std::numeric_limits::max(); |
0 |
| 4584 |
EVT SubVectorVT = Op.getOperand(0).getValueType(); |
0 |
4584 |
EVT SubVectorVT = Op.getOperand(0).getValueType(); |
0 |
| 4585 |
unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements(); |
0 |
4585 |
unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements(); |
0 |
| 4586 |
unsigned NumSubVectors = Op.getNumOperands(); |
0 |
4586 |
unsigned NumSubVectors = Op.getNumOperands(); |
0 |
| 4587 |
for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) { |
0 |
4587 |
for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) { |
0 |
| 4588 |
APInt DemandedSub = |
--- |
4588 |
APInt DemandedSub = |
--- |
| 4589 |
DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts); |
0 |
4589 |
DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts); |
0 |
| 4590 |
if (!DemandedSub) |
0 |
4590 |
if (!DemandedSub) |
0 |
| 4591 |
continue; |
0 |
4591 |
continue; |
0 |
| 4592 |
Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1); |
0 |
4592 |
Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1); |
0 |
| 4593 |
Tmp = std::min(Tmp, Tmp2); |
0 |
4593 |
Tmp = std::min(Tmp, Tmp2); |
0 |
| 4594 |
} |
0 |
4594 |
} |
0 |
| 4595 |
assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); |
0 |
4595 |
assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); |
0 |
| 4596 |
return Tmp; |
0 |
4596 |
return Tmp; |
0 |
| 4597 |
} |
--- |
4597 |
} |
--- |
| 4598 |
case ISD::INSERT_SUBVECTOR: { |
0 |
4598 |
case ISD::INSERT_SUBVECTOR: { |
0 |
| 4599 |
if (VT.isScalableVector()) |
0 |
4599 |
if (VT.isScalableVector()) |
0 |
| 4600 |
break; |
0 |
4600 |
break; |
0 |
| 4601 |
// Demand any elements from the subvector and the remainder from the src its |
--- |
4601 |
// Demand any elements from the subvector and the remainder from the src its |
--- |
| 4602 |
// inserted into. |
--- |
4602 |
// inserted into. |
--- |
| 4603 |
SDValue Src = Op.getOperand(0); |
0 |
4603 |
SDValue Src = Op.getOperand(0); |
0 |
| 4604 |
SDValue Sub = Op.getOperand(1); |
0 |
4604 |
SDValue Sub = Op.getOperand(1); |
0 |
| 4605 |
uint64_t Idx = Op.getConstantOperandVal(2); |
0 |
4605 |
uint64_t Idx = Op.getConstantOperandVal(2); |
0 |
| 4606 |
unsigned NumSubElts = Sub.getValueType().getVectorNumElements(); |
0 |
4606 |
unsigned NumSubElts = Sub.getValueType().getVectorNumElements(); |
0 |
| 4607 |
APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx); |
0 |
4607 |
APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx); |
0 |
| 4608 |
APInt DemandedSrcElts = DemandedElts; |
0 |
4608 |
APInt DemandedSrcElts = DemandedElts; |
0 |
| 4609 |
DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx); |
0 |
4609 |
DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx); |
0 |
| 4610 |
|
--- |
4610 |
|
--- |
| 4611 |
Tmp = std::numeric_limits::max(); |
0 |
4611 |
Tmp = std::numeric_limits::max(); |
0 |
| 4612 |
if (!!DemandedSubElts) { |
0 |
4612 |
if (!!DemandedSubElts) { |
0 |
| 4613 |
Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1); |
0 |
4613 |
Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1); |
0 |
| 4614 |
if (Tmp == 1) |
0 |
4614 |
if (Tmp == 1) |
0 |
| 4615 |
return 1; // early-out |
0 |
4615 |
return 1; // early-out |
0 |
| 4616 |
} |
--- |
4616 |
} |
--- |
| 4617 |
if (!!DemandedSrcElts) { |
0 |
4617 |
if (!!DemandedSrcElts) { |
0 |
| 4618 |
Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1); |
0 |
4618 |
Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1); |
0 |
| 4619 |
Tmp = std::min(Tmp, Tmp2); |
0 |
4619 |
Tmp = std::min(Tmp, Tmp2); |
0 |
| 4620 |
} |
--- |
4620 |
} |
--- |
| 4621 |
assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); |
0 |
4621 |
assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); |
0 |
| 4622 |
return Tmp; |
0 |
4622 |
return Tmp; |
0 |
| 4623 |
} |
0 |
4623 |
} |
0 |
| 4624 |
case ISD::LOAD: { |
0 |
4624 |
case ISD::LOAD: { |
0 |
| 4625 |
LoadSDNode *LD = cast(Op); |
0 |
4625 |
LoadSDNode *LD = cast(Op); |
0 |
| 4626 |
if (const MDNode *Ranges = LD->getRanges()) { |
0 |
4626 |
if (const MDNode *Ranges = LD->getRanges()) { |
0 |
| 4627 |
if (DemandedElts != 1) |
0 |
4627 |
if (DemandedElts != 1) |
0 |
| 4628 |
break; |
0 |
4628 |
break; |
0 |
| 4629 |
|
--- |
4629 |
|
--- |
| 4630 |
ConstantRange CR = getConstantRangeFromMetadata(*Ranges); |
0 |
4630 |
ConstantRange CR = getConstantRangeFromMetadata(*Ranges); |
0 |
| 4631 |
if (VTBits > CR.getBitWidth()) { |
0 |
4631 |
if (VTBits > CR.getBitWidth()) { |
0 |
| 4632 |
switch (LD->getExtensionType()) { |
0 |
4632 |
switch (LD->getExtensionType()) { |
0 |
| 4633 |
case ISD::SEXTLOAD: |
0 |
4633 |
case ISD::SEXTLOAD: |
0 |
| 4634 |
CR = CR.signExtend(VTBits); |
0 |
4634 |
CR = CR.signExtend(VTBits); |
0 |
| 4635 |
break; |
0 |
4635 |
break; |
0 |
| 4636 |
case ISD::ZEXTLOAD: |
0 |
4636 |
case ISD::ZEXTLOAD: |
0 |
| 4637 |
CR = CR.zeroExtend(VTBits); |
0 |
4637 |
CR = CR.zeroExtend(VTBits); |
0 |
| 4638 |
break; |
0 |
4638 |
break; |
0 |
| 4639 |
default: |
0 |
4639 |
default: |
0 |
| 4640 |
break; |
0 |
4640 |
break; |
0 |
| 4641 |
} |
--- |
4641 |
} |
--- |
| 4642 |
} |
--- |
4642 |
} |
--- |
| 4643 |
|
--- |
4643 |
|
--- |
| 4644 |
if (VTBits != CR.getBitWidth()) |
0 |
4644 |
if (VTBits != CR.getBitWidth()) |
0 |
| 4645 |
break; |
0 |
4645 |
break; |
0 |
| 4646 |
return std::min(CR.getSignedMin().getNumSignBits(), |
0 |
4646 |
return std::min(CR.getSignedMin().getNumSignBits(), |
0 |
| 4647 |
CR.getSignedMax().getNumSignBits()); |
0 |
4647 |
CR.getSignedMax().getNumSignBits()); |
0 |
| 4648 |
} |
0 |
4648 |
} |
0 |
| 4649 |
|
--- |
4649 |
|
--- |
| 4650 |
break; |
0 |
4650 |
break; |
0 |
| 4651 |
} |
--- |
4651 |
} |
--- |
| 4652 |
case ISD::ATOMIC_CMP_SWAP: |
0 |
4652 |
case ISD::ATOMIC_CMP_SWAP: |
0 |
| 4653 |
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: |
--- |
4653 |
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: |
--- |
| 4654 |
case ISD::ATOMIC_SWAP: |
--- |
4654 |
case ISD::ATOMIC_SWAP: |
--- |
| 4655 |
case ISD::ATOMIC_LOAD_ADD: |
--- |
4655 |
case ISD::ATOMIC_LOAD_ADD: |
--- |
| 4656 |
case ISD::ATOMIC_LOAD_SUB: |
--- |
4656 |
case ISD::ATOMIC_LOAD_SUB: |
--- |
| 4657 |
case ISD::ATOMIC_LOAD_AND: |
--- |
4657 |
case ISD::ATOMIC_LOAD_AND: |
--- |
| 4658 |
case ISD::ATOMIC_LOAD_CLR: |
--- |
4658 |
case ISD::ATOMIC_LOAD_CLR: |
--- |
| 4659 |
case ISD::ATOMIC_LOAD_OR: |
--- |
4659 |
case ISD::ATOMIC_LOAD_OR: |
--- |
| 4660 |
case ISD::ATOMIC_LOAD_XOR: |
--- |
4660 |
case ISD::ATOMIC_LOAD_XOR: |
--- |
| 4661 |
case ISD::ATOMIC_LOAD_NAND: |
--- |
4661 |
case ISD::ATOMIC_LOAD_NAND: |
--- |
| 4662 |
case ISD::ATOMIC_LOAD_MIN: |
--- |
4662 |
case ISD::ATOMIC_LOAD_MIN: |
--- |
| 4663 |
case ISD::ATOMIC_LOAD_MAX: |
--- |
4663 |
case ISD::ATOMIC_LOAD_MAX: |
--- |
| 4664 |
case ISD::ATOMIC_LOAD_UMIN: |
--- |
4664 |
case ISD::ATOMIC_LOAD_UMIN: |
--- |
| 4665 |
case ISD::ATOMIC_LOAD_UMAX: |
--- |
4665 |
case ISD::ATOMIC_LOAD_UMAX: |
--- |
| 4666 |
case ISD::ATOMIC_LOAD: { |
--- |
4666 |
case ISD::ATOMIC_LOAD: { |
--- |
| 4667 |
Tmp = cast(Op)->getMemoryVT().getScalarSizeInBits(); |
0 |
4667 |
Tmp = cast(Op)->getMemoryVT().getScalarSizeInBits(); |
0 |
| 4668 |
// If we are looking at the loaded value. |
--- |
4668 |
// If we are looking at the loaded value. |
--- |
| 4669 |
if (Op.getResNo() == 0) { |
0 |
4669 |
if (Op.getResNo() == 0) { |
0 |
| 4670 |
if (Tmp == VTBits) |
0 |
4670 |
if (Tmp == VTBits) |
0 |
| 4671 |
return 1; // early-out |
0 |
4671 |
return 1; // early-out |
0 |
| 4672 |
if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND) |
0 |
4672 |
if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND) |
0 |
| 4673 |
return VTBits - Tmp + 1; |
0 |
4673 |
return VTBits - Tmp + 1; |
0 |
| 4674 |
if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND) |
0 |
4674 |
if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND) |
0 |
| 4675 |
return VTBits - Tmp; |
0 |
4675 |
return VTBits - Tmp; |
0 |
| 4676 |
} |
--- |
4676 |
} |
--- |
| 4677 |
break; |
0 |
4677 |
break; |
0 |
| 4678 |
} |
--- |
4678 |
} |
--- |
| 4679 |
} |
--- |
4679 |
} |
--- |
| 4680 |
|
--- |
4680 |
|
--- |
| 4681 |
// If we are looking at the loaded value of the SDNode. |
--- |
4681 |
// If we are looking at the loaded value of the SDNode. |
--- |
| 4682 |
if (Op.getResNo() == 0) { |
0 |
4682 |
if (Op.getResNo() == 0) { |
0 |
| 4683 |
// Handle LOADX separately here. EXTLOAD case will fallthrough. |
--- |
4683 |
// Handle LOADX separately here. EXTLOAD case will fallthrough. |
--- |
| 4684 |
if (LoadSDNode *LD = dyn_cast(Op)) { |
0 |
4684 |
if (LoadSDNode *LD = dyn_cast(Op)) { |
0 |
| 4685 |
unsigned ExtType = LD->getExtensionType(); |
0 |
4685 |
unsigned ExtType = LD->getExtensionType(); |
0 |
| 4686 |
switch (ExtType) { |
0 |
4686 |
switch (ExtType) { |
0 |
| 4687 |
default: break; |
0 |
4687 |
default: break; |
0 |
| 4688 |
case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known. |
0 |
4688 |
case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known. |
0 |
| 4689 |
Tmp = LD->getMemoryVT().getScalarSizeInBits(); |
0 |
4689 |
Tmp = LD->getMemoryVT().getScalarSizeInBits(); |
0 |
| 4690 |
return VTBits - Tmp + 1; |
0 |
4690 |
return VTBits - Tmp + 1; |
0 |
| 4691 |
case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known. |
0 |
4691 |
case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known. |
0 |
| 4692 |
Tmp = LD->getMemoryVT().getScalarSizeInBits(); |
0 |
4692 |
Tmp = LD->getMemoryVT().getScalarSizeInBits(); |
0 |
| 4693 |
return VTBits - Tmp; |
0 |
4693 |
return VTBits - Tmp; |
0 |
| 4694 |
case ISD::NON_EXTLOAD: |
0 |
4694 |
case ISD::NON_EXTLOAD: |
0 |
| 4695 |
if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) { |
0 |
4695 |
if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) { |
0 |
| 4696 |
// We only need to handle vectors - computeKnownBits should handle |
--- |
4696 |
// We only need to handle vectors - computeKnownBits should handle |
--- |
| 4697 |
// scalar cases. |
--- |
4697 |
// scalar cases. |
--- |
| 4698 |
Type *CstTy = Cst->getType(); |
0 |
4698 |
Type *CstTy = Cst->getType(); |
0 |
| 4699 |
if (CstTy->isVectorTy() && !VT.isScalableVector() && |
0 |
4699 |
if (CstTy->isVectorTy() && !VT.isScalableVector() && |
0 |
| 4700 |
(NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() && |
0 |
4700 |
(NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() && |
0 |
| 4701 |
VTBits == CstTy->getScalarSizeInBits()) { |
0 |
4701 |
VTBits == CstTy->getScalarSizeInBits()) { |
0 |
| 4702 |
Tmp = VTBits; |
0 |
4702 |
Tmp = VTBits; |
0 |
| 4703 |
for (unsigned i = 0; i != NumElts; ++i) { |
0 |
4703 |
for (unsigned i = 0; i != NumElts; ++i) { |
0 |
| 4704 |
if (!DemandedElts[i]) |
0 |
4704 |
if (!DemandedElts[i]) |
0 |
| 4705 |
continue; |
0 |
4705 |
continue; |
0 |
| 4706 |
if (Constant *Elt = Cst->getAggregateElement(i)) { |
0 |
4706 |
if (Constant *Elt = Cst->getAggregateElement(i)) { |
0 |
| 4707 |
if (auto *CInt = dyn_cast(Elt)) { |
0 |
4707 |
if (auto *CInt = dyn_cast(Elt)) { |
0 |
| 4708 |
const APInt &Value = CInt->getValue(); |
0 |
4708 |
const APInt &Value = CInt->getValue(); |
0 |
| 4709 |
Tmp = std::min(Tmp, Value.getNumSignBits()); |
0 |
4709 |
Tmp = std::min(Tmp, Value.getNumSignBits()); |
0 |
| 4710 |
continue; |
0 |
4710 |
continue; |
0 |
| 4711 |
} |
0 |
4711 |
} |
0 |
| 4712 |
if (auto *CFP = dyn_cast(Elt)) { |
0 |
4712 |
if (auto *CFP = dyn_cast(Elt)) { |
0 |
| 4713 |
APInt Value = CFP->getValueAPF().bitcastToAPInt(); |
0 |
4713 |
APInt Value = CFP->getValueAPF().bitcastToAPInt(); |
0 |
| 4714 |
Tmp = std::min(Tmp, Value.getNumSignBits()); |
0 |
4714 |
Tmp = std::min(Tmp, Value.getNumSignBits()); |
0 |
| 4715 |
continue; |
0 |
4715 |
continue; |
0 |
| 4716 |
} |
0 |
4716 |
} |
0 |
| 4717 |
} |
--- |
4717 |
} |
--- |
| 4718 |
// Unknown type. Conservatively assume no bits match sign bit. |
--- |
4718 |
// Unknown type. Conservatively assume no bits match sign bit. |
--- |
| 4719 |
return 1; |
0 |
4719 |
return 1; |
0 |
| 4720 |
} |
--- |
4720 |
} |
--- |
| 4721 |
return Tmp; |
0 |
4721 |
return Tmp; |
0 |
| 4722 |
} |
--- |
4722 |
} |
--- |
| 4723 |
} |
--- |
4723 |
} |
--- |
| 4724 |
break; |
0 |
4724 |
break; |
0 |
| 4725 |
} |
--- |
4725 |
} |
--- |
| 4726 |
} |
--- |
4726 |
} |
--- |
| 4727 |
} |
--- |
4727 |
} |
--- |
| 4728 |
|
--- |
4728 |
|
--- |
| 4729 |
// Allow the target to implement this method for its nodes. |
--- |
4729 |
// Allow the target to implement this method for its nodes. |
--- |
| 4730 |
if (Opcode >= ISD::BUILTIN_OP_END || |
0 |
4730 |
if (Opcode >= ISD::BUILTIN_OP_END || |
0 |
| 4731 |
Opcode == ISD::INTRINSIC_WO_CHAIN || |
0 |
4731 |
Opcode == ISD::INTRINSIC_WO_CHAIN || |
0 |
| 4732 |
Opcode == ISD::INTRINSIC_W_CHAIN || |
0 |
4732 |
Opcode == ISD::INTRINSIC_W_CHAIN || |
0 |
| 4733 |
Opcode == ISD::INTRINSIC_VOID) { |
--- |
4733 |
Opcode == ISD::INTRINSIC_VOID) { |
--- |
| 4734 |
// TODO: This can probably be removed once target code is audited. This |
--- |
4734 |
// TODO: This can probably be removed once target code is audited. This |
--- |
| 4735 |
// is here purely to reduce patch size and review complexity. |
--- |
4735 |
// is here purely to reduce patch size and review complexity. |
--- |
| 4736 |
if (!VT.isScalableVector()) { |
0 |
4736 |
if (!VT.isScalableVector()) { |
0 |
| 4737 |
unsigned NumBits = |
--- |
4737 |
unsigned NumBits = |
--- |
| 4738 |
TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth); |
0 |
4738 |
TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth); |
0 |
| 4739 |
if (NumBits > 1) |
0 |
4739 |
if (NumBits > 1) |
0 |
| 4740 |
FirstAnswer = std::max(FirstAnswer, NumBits); |
0 |
4740 |
FirstAnswer = std::max(FirstAnswer, NumBits); |
0 |
| 4741 |
} |
--- |
4741 |
} |
--- |
| 4742 |
} |
--- |
4742 |
} |
--- |
| 4743 |
|
--- |
4743 |
|
--- |
| 4744 |
// Finally, if we can prove that the top bits of the result are 0's or 1's, |
--- |
4744 |
// Finally, if we can prove that the top bits of the result are 0's or 1's, |
--- |
| 4745 |
// use this information. |
--- |
4745 |
// use this information. |
--- |
| 4746 |
KnownBits Known = computeKnownBits(Op, DemandedElts, Depth); |
0 |
4746 |
KnownBits Known = computeKnownBits(Op, DemandedElts, Depth); |
0 |
| 4747 |
return std::max(FirstAnswer, Known.countMinSignBits()); |
0 |
4747 |
return std::max(FirstAnswer, Known.countMinSignBits()); |
0 |
| 4748 |
} |
--- |
4748 |
} |
--- |
| 4749 |
|
--- |
4749 |
|
--- |
| 4750 |
unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op, |
0 |
4750 |
unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op, |
0 |
| 4751 |
unsigned Depth) const { |
--- |
4751 |
unsigned Depth) const { |
--- |
| 4752 |
unsigned SignBits = ComputeNumSignBits(Op, Depth); |
0 |
4752 |
unsigned SignBits = ComputeNumSignBits(Op, Depth); |
0 |
| 4753 |
return Op.getScalarValueSizeInBits() - SignBits + 1; |
0 |
4753 |
return Op.getScalarValueSizeInBits() - SignBits + 1; |
0 |
| 4754 |
} |
--- |
4754 |
} |
--- |
| 4755 |
|
--- |
4755 |
|
--- |
| 4756 |
unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op, |
0 |
4756 |
unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op, |
0 |
| 4757 |
const APInt &DemandedElts, |
--- |
4757 |
const APInt &DemandedElts, |
--- |
| 4758 |
unsigned Depth) const { |
--- |
4758 |
unsigned Depth) const { |
--- |
| 4759 |
unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth); |
0 |
4759 |
unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth); |
0 |
| 4760 |
return Op.getScalarValueSizeInBits() - SignBits + 1; |
0 |
4760 |
return Op.getScalarValueSizeInBits() - SignBits + 1; |
0 |
| 4761 |
} |
--- |
4761 |
} |
--- |
| 4762 |
|
--- |
4762 |
|
--- |
| 4763 |
bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly, |
0 |
4763 |
bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly, |
0 |
| 4764 |
unsigned Depth) const { |
--- |
4764 |
unsigned Depth) const { |
--- |
| 4765 |
// Early out for FREEZE. |
--- |
4765 |
// Early out for FREEZE. |
--- |
| 4766 |
if (Op.getOpcode() == ISD::FREEZE) |
0 |
4766 |
if (Op.getOpcode() == ISD::FREEZE) |
0 |
| 4767 |
return true; |
0 |
4767 |
return true; |
0 |
| 4768 |
|
--- |
4768 |
|
--- |
| 4769 |
// TODO: Assume we don't know anything for now. |
--- |
4769 |
// TODO: Assume we don't know anything for now. |
--- |
| 4770 |
EVT VT = Op.getValueType(); |
0 |
4770 |
EVT VT = Op.getValueType(); |
0 |
| 4771 |
if (VT.isScalableVector()) |
0 |
4771 |
if (VT.isScalableVector()) |
0 |
| 4772 |
return false; |
0 |
4772 |
return false; |
0 |
| 4773 |
|
--- |
4773 |
|
--- |
| 4774 |
APInt DemandedElts = VT.isVector() |
0 |
4774 |
APInt DemandedElts = VT.isVector() |
0 |
| 4775 |
? APInt::getAllOnes(VT.getVectorNumElements()) |
--- |
4775 |
? APInt::getAllOnes(VT.getVectorNumElements()) |
--- |
| 4776 |
: APInt(1, 1); |
0 |
4776 |
: APInt(1, 1); |
0 |
| 4777 |
return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth); |
0 |
4777 |
return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth); |
0 |
| 4778 |
} |
0 |
4778 |
} |
0 |
| 4779 |
|
--- |
4779 |
|
--- |
| 4780 |
bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, |
0 |
4780 |
bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, |
0 |
| 4781 |
const APInt &DemandedElts, |
--- |
4781 |
const APInt &DemandedElts, |
--- |
| 4782 |
bool PoisonOnly, |
--- |
4782 |
bool PoisonOnly, |
--- |
| 4783 |
unsigned Depth) const { |
--- |
4783 |
unsigned Depth) const { |
--- |
| 4784 |
unsigned Opcode = Op.getOpcode(); |
0 |
4784 |
unsigned Opcode = Op.getOpcode(); |
0 |
| 4785 |
|
--- |
4785 |
|
--- |
| 4786 |
// Early out for FREEZE. |
--- |
4786 |
// Early out for FREEZE. |
--- |
| 4787 |
if (Opcode == ISD::FREEZE) |
0 |
4787 |
if (Opcode == ISD::FREEZE) |
0 |
| 4788 |
return true; |
0 |
4788 |
return true; |
0 |
| 4789 |
|
--- |
4789 |
|
--- |
| 4790 |
if (Depth >= MaxRecursionDepth) |
0 |
4790 |
if (Depth >= MaxRecursionDepth) |
0 |
| 4791 |
return false; // Limit search depth. |
0 |
4791 |
return false; // Limit search depth. |
0 |
| 4792 |
|
--- |
4792 |
|
--- |
| 4793 |
if (isIntOrFPConstant(Op)) |
0 |
4793 |
if (isIntOrFPConstant(Op)) |
0 |
| 4794 |
return true; |
0 |
4794 |
return true; |
0 |
| 4795 |
|
--- |
4795 |
|
--- |
| 4796 |
switch (Opcode) { |
0 |
4796 |
switch (Opcode) { |
0 |
| 4797 |
case ISD::VALUETYPE: |
0 |
4797 |
case ISD::VALUETYPE: |
0 |
| 4798 |
case ISD::FrameIndex: |
--- |
4798 |
case ISD::FrameIndex: |
--- |
| 4799 |
case ISD::TargetFrameIndex: |
--- |
4799 |
case ISD::TargetFrameIndex: |
--- |
| 4800 |
return true; |
0 |
4800 |
return true; |
0 |
| 4801 |
|
--- |
4801 |
|
--- |
| 4802 |
case ISD::UNDEF: |
0 |
4802 |
case ISD::UNDEF: |
0 |
| 4803 |
return PoisonOnly; |
0 |
4803 |
return PoisonOnly; |
0 |
| 4804 |
|
--- |
4804 |
|
--- |
| 4805 |
case ISD::BUILD_VECTOR: |
0 |
4805 |
case ISD::BUILD_VECTOR: |
0 |
| 4806 |
// NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements - |
--- |
4806 |
// NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements - |
--- |
| 4807 |
// this shouldn't affect the result. |
--- |
4807 |
// this shouldn't affect the result. |
--- |
| 4808 |
for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) { |
0 |
4808 |
for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) { |
0 |
| 4809 |
if (!DemandedElts[i]) |
0 |
4809 |
if (!DemandedElts[i]) |
0 |
| 4810 |
continue; |
0 |
4810 |
continue; |
0 |
| 4811 |
if (!isGuaranteedNotToBeUndefOrPoison(Op.getOperand(i), PoisonOnly, |
0 |
4811 |
if (!isGuaranteedNotToBeUndefOrPoison(Op.getOperand(i), PoisonOnly, |
0 |
| 4812 |
Depth + 1)) |
--- |
4812 |
Depth + 1)) |
--- |
| 4813 |
return false; |
0 |
4813 |
return false; |
0 |
| 4814 |
} |
--- |
4814 |
} |
--- |
| 4815 |
return true; |
0 |
4815 |
return true; |
0 |
| 4816 |
|
--- |
4816 |
|
--- |
| 4817 |
// TODO: Search for noundef attributes from library functions. |
--- |
4817 |
// TODO: Search for noundef attributes from library functions. |
--- |
| 4818 |
|
--- |
4818 |
|
--- |
| 4819 |
// TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef. |
--- |
4819 |
// TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef. |
--- |
| 4820 |
|
--- |
4820 |
|
--- |
| 4821 |
default: |
0 |
4821 |
default: |
0 |
| 4822 |
// Allow the target to implement this method for its nodes. |
--- |
4822 |
// Allow the target to implement this method for its nodes. |
--- |
| 4823 |
if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN || |
0 |
4823 |
if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN || |
0 |
| 4824 |
Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) |
0 |
4824 |
Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) |
0 |
| 4825 |
return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode( |
0 |
4825 |
return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode( |
0 |
| 4826 |
Op, DemandedElts, *this, PoisonOnly, Depth); |
0 |
4826 |
Op, DemandedElts, *this, PoisonOnly, Depth); |
0 |
| 4827 |
break; |
0 |
4827 |
break; |
0 |
| 4828 |
} |
--- |
4828 |
} |
--- |
| 4829 |
|
--- |
4829 |
|
--- |
| 4830 |
// If Op can't create undef/poison and none of its operands are undef/poison |
--- |
4830 |
// If Op can't create undef/poison and none of its operands are undef/poison |
--- |
| 4831 |
// then Op is never undef/poison. |
--- |
4831 |
// then Op is never undef/poison. |
--- |
| 4832 |
// NOTE: TargetNodes should handle this in themselves in |
--- |
4832 |
// NOTE: TargetNodes should handle this in themselves in |
--- |
| 4833 |
// isGuaranteedNotToBeUndefOrPoisonForTargetNode. |
--- |
4833 |
// isGuaranteedNotToBeUndefOrPoisonForTargetNode. |
--- |
| 4834 |
return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true, |
0 |
4834 |
return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true, |
0 |
| 4835 |
Depth) && |
0 |
4835 |
Depth) && |
0 |
| 4836 |
all_of(Op->ops(), [&](SDValue V) { |
0 |
4836 |
all_of(Op->ops(), [&](SDValue V) { |
0 |
| 4837 |
return isGuaranteedNotToBeUndefOrPoison(V, PoisonOnly, Depth + 1); |
0 |
4837 |
return isGuaranteedNotToBeUndefOrPoison(V, PoisonOnly, Depth + 1); |
0 |
| 4838 |
}); |
0 |
4838 |
}); |
0 |
| 4839 |
} |
--- |
4839 |
} |
--- |
| 4840 |
|
--- |
4840 |
|
--- |
| 4841 |
bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, bool PoisonOnly, |
0 |
4841 |
bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, bool PoisonOnly, |
0 |
| 4842 |
bool ConsiderFlags, |
--- |
4842 |
bool ConsiderFlags, |
--- |
| 4843 |
unsigned Depth) const { |
--- |
4843 |
unsigned Depth) const { |
--- |
| 4844 |
// TODO: Assume we don't know anything for now. |
--- |
4844 |
// TODO: Assume we don't know anything for now. |
--- |
| 4845 |
EVT VT = Op.getValueType(); |
0 |
4845 |
EVT VT = Op.getValueType(); |
0 |
| 4846 |
if (VT.isScalableVector()) |
0 |
4846 |
if (VT.isScalableVector()) |
0 |
| 4847 |
return true; |
0 |
4847 |
return true; |
0 |
| 4848 |
|
--- |
4848 |
|
--- |
| 4849 |
APInt DemandedElts = VT.isVector() |
0 |
4849 |
APInt DemandedElts = VT.isVector() |
0 |
| 4850 |
? APInt::getAllOnes(VT.getVectorNumElements()) |
--- |
4850 |
? APInt::getAllOnes(VT.getVectorNumElements()) |
--- |
| 4851 |
: APInt(1, 1); |
0 |
4851 |
: APInt(1, 1); |
0 |
| 4852 |
return canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly, ConsiderFlags, |
0 |
4852 |
return canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly, ConsiderFlags, |
0 |
| 4853 |
Depth); |
0 |
4853 |
Depth); |
0 |
| 4854 |
} |
0 |
4854 |
} |
0 |
| 4855 |
|
--- |
4855 |
|
--- |
| 4856 |
bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, |
0 |
4856 |
bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, |
0 |
| 4857 |
bool PoisonOnly, bool ConsiderFlags, |
--- |
4857 |
bool PoisonOnly, bool ConsiderFlags, |
--- |
| 4858 |
unsigned Depth) const { |
--- |
4858 |
unsigned Depth) const { |
--- |
| 4859 |
// TODO: Assume we don't know anything for now. |
--- |
4859 |
// TODO: Assume we don't know anything for now. |
--- |
| 4860 |
EVT VT = Op.getValueType(); |
0 |
4860 |
EVT VT = Op.getValueType(); |
0 |
| 4861 |
if (VT.isScalableVector()) |
0 |
4861 |
if (VT.isScalableVector()) |
0 |
| 4862 |
return true; |
0 |
4862 |
return true; |
0 |
| 4863 |
|
--- |
4863 |
|
--- |
| 4864 |
unsigned Opcode = Op.getOpcode(); |
0 |
4864 |
unsigned Opcode = Op.getOpcode(); |
0 |
| 4865 |
switch (Opcode) { |
0 |
4865 |
switch (Opcode) { |
0 |
| 4866 |
case ISD::AssertSext: |
0 |
4866 |
case ISD::AssertSext: |
0 |
| 4867 |
case ISD::AssertZext: |
--- |
4867 |
case ISD::AssertZext: |
--- |
| 4868 |
case ISD::FREEZE: |
--- |
4868 |
case ISD::FREEZE: |
--- |
| 4869 |
case ISD::CONCAT_VECTORS: |
--- |
4869 |
case ISD::CONCAT_VECTORS: |
--- |
| 4870 |
case ISD::INSERT_SUBVECTOR: |
--- |
4870 |
case ISD::INSERT_SUBVECTOR: |
--- |
| 4871 |
case ISD::AND: |
--- |
4871 |
case ISD::AND: |
--- |
| 4872 |
case ISD::OR: |
--- |
4872 |
case ISD::OR: |
--- |
| 4873 |
case ISD::XOR: |
--- |
4873 |
case ISD::XOR: |
--- |
| 4874 |
case ISD::ROTL: |
--- |
4874 |
case ISD::ROTL: |
--- |
| 4875 |
case ISD::ROTR: |
--- |
4875 |
case ISD::ROTR: |
--- |
| 4876 |
case ISD::FSHL: |
--- |
4876 |
case ISD::FSHL: |
--- |
| 4877 |
case ISD::FSHR: |
--- |
4877 |
case ISD::FSHR: |
--- |
| 4878 |
case ISD::BSWAP: |
--- |
4878 |
case ISD::BSWAP: |
--- |
| 4879 |
case ISD::CTPOP: |
--- |
4879 |
case ISD::CTPOP: |
--- |
| 4880 |
case ISD::BITREVERSE: |
--- |
4880 |
case ISD::BITREVERSE: |
--- |
| 4881 |
case ISD::PARITY: |
--- |
4881 |
case ISD::PARITY: |
--- |
| 4882 |
case ISD::SIGN_EXTEND: |
--- |
4882 |
case ISD::SIGN_EXTEND: |
--- |
| 4883 |
case ISD::ZERO_EXTEND: |
--- |
4883 |
case ISD::ZERO_EXTEND: |
--- |
| 4884 |
case ISD::TRUNCATE: |
--- |
4884 |
case ISD::TRUNCATE: |
--- |
| 4885 |
case ISD::SIGN_EXTEND_INREG: |
--- |
4885 |
case ISD::SIGN_EXTEND_INREG: |
--- |
| 4886 |
case ISD::SIGN_EXTEND_VECTOR_INREG: |
--- |
4886 |
case ISD::SIGN_EXTEND_VECTOR_INREG: |
--- |
| 4887 |
case ISD::ZERO_EXTEND_VECTOR_INREG: |
--- |
4887 |
case ISD::ZERO_EXTEND_VECTOR_INREG: |
--- |
| 4888 |
case ISD::BITCAST: |
--- |
4888 |
case ISD::BITCAST: |
--- |
| 4889 |
case ISD::BUILD_VECTOR: |
--- |
4889 |
case ISD::BUILD_VECTOR: |
--- |
| 4890 |
case ISD::BUILD_PAIR: |
--- |
4890 |
case ISD::BUILD_PAIR: |
--- |
| 4891 |
return false; |
0 |
4891 |
return false; |
0 |
| 4892 |
|
--- |
4892 |
|
--- |
| 4893 |
case ISD::ADD: |
0 |
4893 |
case ISD::ADD: |
0 |
| 4894 |
case ISD::SUB: |
--- |
4894 |
case ISD::SUB: |
--- |
| 4895 |
case ISD::MUL: |
--- |
4895 |
case ISD::MUL: |
--- |
| 4896 |
// Matches hasPoisonGeneratingFlags(). |
--- |
4896 |
// Matches hasPoisonGeneratingFlags(). |
--- |
| 4897 |
return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() || |
0 |
4897 |
return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() || |
0 |
| 4898 |
Op->getFlags().hasNoUnsignedWrap()); |
0 |
4898 |
Op->getFlags().hasNoUnsignedWrap()); |
0 |
| 4899 |
|
--- |
4899 |
|
--- |
| 4900 |
case ISD::SHL: |
0 |
4900 |
case ISD::SHL: |
0 |
| 4901 |
// If the max shift amount isn't in range, then the shift can create poison. |
--- |
4901 |
// If the max shift amount isn't in range, then the shift can create poison. |
--- |
| 4902 |
if (!getValidMaximumShiftAmountConstant(Op, DemandedElts)) |
0 |
4902 |
if (!getValidMaximumShiftAmountConstant(Op, DemandedElts)) |
0 |
| 4903 |
return true; |
0 |
4903 |
return true; |
0 |
| 4904 |
|
--- |
4904 |
|
--- |
| 4905 |
// Matches hasPoisonGeneratingFlags(). |
--- |
4905 |
// Matches hasPoisonGeneratingFlags(). |
--- |
| 4906 |
return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() || |
0 |
4906 |
return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() || |
0 |
| 4907 |
Op->getFlags().hasNoUnsignedWrap()); |
0 |
4907 |
Op->getFlags().hasNoUnsignedWrap()); |
0 |
| 4908 |
|
--- |
4908 |
|
--- |
| 4909 |
case ISD::INSERT_VECTOR_ELT:{ |
0 |
4909 |
case ISD::INSERT_VECTOR_ELT:{ |
0 |
| 4910 |
// Ensure that the element index is in bounds. |
--- |
4910 |
// Ensure that the element index is in bounds. |
--- |
| 4911 |
EVT VecVT = Op.getOperand(0).getValueType(); |
0 |
4911 |
EVT VecVT = Op.getOperand(0).getValueType(); |
0 |
| 4912 |
KnownBits KnownIdx = computeKnownBits(Op.getOperand(2), Depth + 1); |
0 |
4912 |
KnownBits KnownIdx = computeKnownBits(Op.getOperand(2), Depth + 1); |
0 |
| 4913 |
return KnownIdx.getMaxValue().uge(VecVT.getVectorMinNumElements()); |
0 |
4913 |
return KnownIdx.getMaxValue().uge(VecVT.getVectorMinNumElements()); |
0 |
| 4914 |
} |
0 |
4914 |
} |
0 |
| 4915 |
|
--- |
4915 |
|
--- |
| 4916 |
default: |
0 |
4916 |
default: |
0 |
| 4917 |
// Allow the target to implement this method for its nodes. |
--- |
4917 |
// Allow the target to implement this method for its nodes. |
--- |
| 4918 |
if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN || |
0 |
4918 |
if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN || |
0 |
| 4919 |
Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) |
0 |
4919 |
Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) |
0 |
| 4920 |
return TLI->canCreateUndefOrPoisonForTargetNode( |
0 |
4920 |
return TLI->canCreateUndefOrPoisonForTargetNode( |
0 |
| 4921 |
Op, DemandedElts, *this, PoisonOnly, ConsiderFlags, Depth); |
0 |
4921 |
Op, DemandedElts, *this, PoisonOnly, ConsiderFlags, Depth); |
0 |
| 4922 |
break; |
0 |
4922 |
break; |
0 |
| 4923 |
} |
--- |
4923 |
} |
--- |
| 4924 |
|
--- |
4924 |
|
--- |
| 4925 |
// Be conservative and return true. |
--- |
4925 |
// Be conservative and return true. |
--- |
| 4926 |
return true; |
0 |
4926 |
return true; |
0 |
| 4927 |
} |
--- |
4927 |
} |
--- |
| 4928 |
|
--- |
4928 |
|
--- |
| 4929 |
bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { |
0 |
4929 |
bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { |
0 |
| 4930 |
if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || |
0 |
4930 |
if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || |
0 |
| 4931 |
!isa(Op.getOperand(1))) |
0 |
4931 |
!isa(Op.getOperand(1))) |
0 |
| 4932 |
return false; |
0 |
4932 |
return false; |
0 |
| 4933 |
|
--- |
4933 |
|
--- |
| 4934 |
if (Op.getOpcode() == ISD::OR && |
0 |
4934 |
if (Op.getOpcode() == ISD::OR && |
0 |
| 4935 |
!MaskedValueIsZero(Op.getOperand(0), Op.getConstantOperandAPInt(1))) |
0 |
4935 |
!MaskedValueIsZero(Op.getOperand(0), Op.getConstantOperandAPInt(1))) |
0 |
| 4936 |
return false; |
0 |
4936 |
return false; |
0 |
| 4937 |
|
--- |
4937 |
|
--- |
| 4938 |
return true; |
0 |
4938 |
return true; |
0 |
| 4939 |
} |
--- |
4939 |
} |
--- |
| 4940 |
|
--- |
4940 |
|
--- |
| 4941 |
bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const { |
0 |
4941 |
bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const { |
0 |
| 4942 |
// If we're told that NaNs won't happen, assume they won't. |
--- |
4942 |
// If we're told that NaNs won't happen, assume they won't. |
--- |
| 4943 |
if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs()) |
0 |
4943 |
if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs()) |
0 |
| 4944 |
return true; |
0 |
4944 |
return true; |
0 |
| 4945 |
|
--- |
4945 |
|
--- |
| 4946 |
if (Depth >= MaxRecursionDepth) |
0 |
4946 |
if (Depth >= MaxRecursionDepth) |
0 |
| 4947 |
return false; // Limit search depth. |
0 |
4947 |
return false; // Limit search depth. |
0 |
| 4948 |
|
--- |
4948 |
|
--- |
| 4949 |
// If the value is a constant, we can obviously see if it is a NaN or not. |
--- |
4949 |
// If the value is a constant, we can obviously see if it is a NaN or not. |
--- |
| 4950 |
if (const ConstantFPSDNode *C = dyn_cast(Op)) { |
0 |
4950 |
if (const ConstantFPSDNode *C = dyn_cast(Op)) { |
0 |
| 4951 |
return !C->getValueAPF().isNaN() || |
0 |
4951 |
return !C->getValueAPF().isNaN() || |
0 |
| 4952 |
(SNaN && !C->getValueAPF().isSignaling()); |
0 |
4952 |
(SNaN && !C->getValueAPF().isSignaling()); |
0 |
| 4953 |
} |
--- |
4953 |
} |
--- |
| 4954 |
|
--- |
4954 |
|
--- |
| 4955 |
unsigned Opcode = Op.getOpcode(); |
0 |
4955 |
unsigned Opcode = Op.getOpcode(); |
0 |
| 4956 |
switch (Opcode) { |
0 |
4956 |
switch (Opcode) { |
0 |
| 4957 |
case ISD::FADD: |
0 |
4957 |
case ISD::FADD: |
0 |
| 4958 |
case ISD::FSUB: |
--- |
4958 |
case ISD::FSUB: |
--- |
| 4959 |
case ISD::FMUL: |
--- |
4959 |
case ISD::FMUL: |
--- |
| 4960 |
case ISD::FDIV: |
--- |
4960 |
case ISD::FDIV: |
--- |
| 4961 |
case ISD::FREM: |
--- |
4961 |
case ISD::FREM: |
--- |
| 4962 |
case ISD::FSIN: |
--- |
4962 |
case ISD::FSIN: |
--- |
| 4963 |
case ISD::FCOS: |
--- |
4963 |
case ISD::FCOS: |
--- |
| 4964 |
case ISD::FMA: |
--- |
4964 |
case ISD::FMA: |
--- |
| 4965 |
case ISD::FMAD: { |
--- |
4965 |
case ISD::FMAD: { |
--- |
| 4966 |
if (SNaN) |
0 |
4966 |
if (SNaN) |
0 |
| 4967 |
return true; |
0 |
4967 |
return true; |
0 |
| 4968 |
// TODO: Need isKnownNeverInfinity |
--- |
4968 |
// TODO: Need isKnownNeverInfinity |
--- |
| 4969 |
return false; |
0 |
4969 |
return false; |
0 |
| 4970 |
} |
--- |
4970 |
} |
--- |
| 4971 |
case ISD::FCANONICALIZE: |
0 |
4971 |
case ISD::FCANONICALIZE: |
0 |
| 4972 |
case ISD::FEXP: |
--- |
4972 |
case ISD::FEXP: |
--- |
| 4973 |
case ISD::FEXP2: |
--- |
4973 |
case ISD::FEXP2: |
--- |
| 4974 |
case ISD::FTRUNC: |
--- |
4974 |
case ISD::FTRUNC: |
--- |
| 4975 |
case ISD::FFLOOR: |
--- |
4975 |
case ISD::FFLOOR: |
--- |
| 4976 |
case ISD::FCEIL: |
--- |
4976 |
case ISD::FCEIL: |
--- |
| 4977 |
case ISD::FROUND: |
--- |
4977 |
case ISD::FROUND: |
--- |
| 4978 |
case ISD::FROUNDEVEN: |
--- |
4978 |
case ISD::FROUNDEVEN: |
--- |
| 4979 |
case ISD::FRINT: |
--- |
4979 |
case ISD::FRINT: |
--- |
| 4980 |
case ISD::FNEARBYINT: |
--- |
4980 |
case ISD::FNEARBYINT: |
--- |
| 4981 |
case ISD::FLDEXP: { |
--- |
4981 |
case ISD::FLDEXP: { |
--- |
| 4982 |
if (SNaN) |
0 |
4982 |
if (SNaN) |
0 |
| 4983 |
return true; |
0 |
4983 |
return true; |
0 |
| 4984 |
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); |
0 |
4984 |
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); |
0 |
| 4985 |
} |
--- |
4985 |
} |
--- |
| 4986 |
case ISD::FABS: |
0 |
4986 |
case ISD::FABS: |
0 |
| 4987 |
case ISD::FNEG: |
--- |
4987 |
case ISD::FNEG: |
--- |
| 4988 |
case ISD::FCOPYSIGN: { |
--- |
4988 |
case ISD::FCOPYSIGN: { |
--- |
| 4989 |
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); |
0 |
4989 |
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); |
0 |
| 4990 |
} |
--- |
4990 |
} |
--- |
| 4991 |
case ISD::SELECT: |
0 |
4991 |
case ISD::SELECT: |
0 |
| 4992 |
return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) && |
0 |
4992 |
return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) && |
0 |
| 4993 |
isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1); |
0 |
4993 |
isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1); |
0 |
| 4994 |
case ISD::FP_EXTEND: |
0 |
4994 |
case ISD::FP_EXTEND: |
0 |
| 4995 |
case ISD::FP_ROUND: { |
--- |
4995 |
case ISD::FP_ROUND: { |
--- |
| 4996 |
if (SNaN) |
0 |
4996 |
if (SNaN) |
0 |
| 4997 |
return true; |
0 |
4997 |
return true; |
0 |
| 4998 |
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); |
0 |
4998 |
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); |
0 |
| 4999 |
} |
--- |
4999 |
} |
--- |
| 5000 |
case ISD::SINT_TO_FP: |
0 |
5000 |
case ISD::SINT_TO_FP: |
0 |
| 5001 |
case ISD::UINT_TO_FP: |
--- |
5001 |
case ISD::UINT_TO_FP: |
--- |
| 5002 |
return true; |
0 |
5002 |
return true; |
0 |
| 5003 |
case ISD::FSQRT: // Need is known positive |
0 |
5003 |
case ISD::FSQRT: // Need is known positive |
0 |
| 5004 |
case ISD::FLOG: |
--- |
5004 |
case ISD::FLOG: |
--- |
| 5005 |
case ISD::FLOG2: |
--- |
5005 |
case ISD::FLOG2: |
--- |
| 5006 |
case ISD::FLOG10: |
--- |
5006 |
case ISD::FLOG10: |
--- |
| 5007 |
case ISD::FPOWI: |
--- |
5007 |
case ISD::FPOWI: |
--- |
| 5008 |
case ISD::FPOW: { |
--- |
5008 |
case ISD::FPOW: { |
--- |
| 5009 |
if (SNaN) |
0 |
5009 |
if (SNaN) |
0 |
| 5010 |
return true; |
0 |
5010 |
return true; |
0 |
| 5011 |
// TODO: Refine on operand |
--- |
5011 |
// TODO: Refine on operand |
--- |
| 5012 |
return false; |
0 |
5012 |
return false; |
0 |
| 5013 |
} |
--- |
5013 |
} |
--- |
| 5014 |
case ISD::FMINNUM: |
0 |
5014 |
case ISD::FMINNUM: |
0 |
| 5015 |
case ISD::FMAXNUM: { |
--- |
5015 |
case ISD::FMAXNUM: { |
--- |
| 5016 |
// Only one needs to be known not-nan, since it will be returned if the |
--- |
5016 |
// Only one needs to be known not-nan, since it will be returned if the |
--- |
| 5017 |
// other ends up being one. |
--- |
5017 |
// other ends up being one. |
--- |
| 5018 |
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) || |
0 |
5018 |
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) || |
0 |
| 5019 |
isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1); |
0 |
5019 |
isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1); |
0 |
| 5020 |
} |
--- |
5020 |
} |
--- |
| 5021 |
case ISD::FMINNUM_IEEE: |
0 |
5021 |
case ISD::FMINNUM_IEEE: |
0 |
| 5022 |
case ISD::FMAXNUM_IEEE: { |
--- |
5022 |
case ISD::FMAXNUM_IEEE: { |
--- |
| 5023 |
if (SNaN) |
0 |
5023 |
if (SNaN) |
0 |
| 5024 |
return true; |
0 |
5024 |
return true; |
0 |
| 5025 |
// This can return a NaN if either operand is an sNaN, or if both operands |
--- |
5025 |
// This can return a NaN if either operand is an sNaN, or if both operands |
--- |
| 5026 |
// are NaN. |
--- |
5026 |
// are NaN. |
--- |
| 5027 |
return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) && |
0 |
5027 |
return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) && |
0 |
| 5028 |
isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) || |
0 |
5028 |
isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) || |
0 |
| 5029 |
(isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) && |
0 |
5029 |
(isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) && |
0 |
| 5030 |
isKnownNeverSNaN(Op.getOperand(0), Depth + 1)); |
0 |
5030 |
isKnownNeverSNaN(Op.getOperand(0), Depth + 1)); |
0 |
| 5031 |
} |
--- |
5031 |
} |
--- |
| 5032 |
case ISD::FMINIMUM: |
0 |
5032 |
case ISD::FMINIMUM: |
0 |
| 5033 |
case ISD::FMAXIMUM: { |
--- |
5033 |
case ISD::FMAXIMUM: { |
--- |
| 5034 |
// TODO: Does this quiet or return the origina NaN as-is? |
--- |
5034 |
// TODO: Does this quiet or return the origina NaN as-is? |
--- |
| 5035 |
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) && |
0 |
5035 |
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) && |
0 |
| 5036 |
isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1); |
0 |
5036 |
isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1); |
0 |
| 5037 |
} |
--- |
5037 |
} |
--- |
| 5038 |
case ISD::EXTRACT_VECTOR_ELT: { |
0 |
5038 |
case ISD::EXTRACT_VECTOR_ELT: { |
0 |
| 5039 |
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); |
0 |
5039 |
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); |
0 |
| 5040 |
} |
--- |
5040 |
} |
--- |
| 5041 |
case ISD::BUILD_VECTOR: { |
0 |
5041 |
case ISD::BUILD_VECTOR: { |
0 |
| 5042 |
for (const SDValue &Opnd : Op->ops()) |
0 |
5042 |
for (const SDValue &Opnd : Op->ops()) |
0 |
| 5043 |
if (!isKnownNeverNaN(Opnd, SNaN, Depth + 1)) |
0 |
5043 |
if (!isKnownNeverNaN(Opnd, SNaN, Depth + 1)) |
0 |
| 5044 |
return false; |
0 |
5044 |
return false; |
0 |
| 5045 |
return true; |
0 |
5045 |
return true; |
0 |
| 5046 |
} |
--- |
5046 |
} |
--- |
| 5047 |
default: |
0 |
5047 |
default: |
0 |
| 5048 |
if (Opcode >= ISD::BUILTIN_OP_END || |
0 |
5048 |
if (Opcode >= ISD::BUILTIN_OP_END || |
0 |
| 5049 |
Opcode == ISD::INTRINSIC_WO_CHAIN || |
0 |
5049 |
Opcode == ISD::INTRINSIC_WO_CHAIN || |
0 |
| 5050 |
Opcode == ISD::INTRINSIC_W_CHAIN || |
0 |
5050 |
Opcode == ISD::INTRINSIC_W_CHAIN || |
0 |
| 5051 |
Opcode == ISD::INTRINSIC_VOID) { |
--- |
5051 |
Opcode == ISD::INTRINSIC_VOID) { |
--- |
| 5052 |
return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth); |
0 |
5052 |
return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth); |
0 |
| 5053 |
} |
--- |
5053 |
} |
--- |
| 5054 |
|
--- |
5054 |
|
--- |
| 5055 |
return false; |
0 |
5055 |
return false; |
0 |
| 5056 |
} |
--- |
5056 |
} |
--- |
| 5057 |
} |
--- |
5057 |
} |
--- |
| 5058 |
|
--- |
5058 |
|
--- |
| 5059 |
bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const { |
0 |
5059 |
bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const { |
0 |
| 5060 |
assert(Op.getValueType().isFloatingPoint() && |
0 |
5060 |
assert(Op.getValueType().isFloatingPoint() && |
0 |
| 5061 |
"Floating point type expected"); |
--- |
5061 |
"Floating point type expected"); |
--- |
| 5062 |
|
--- |
5062 |
|
--- |
| 5063 |
// If the value is a constant, we can obviously see if it is a zero or not. |
--- |
5063 |
// If the value is a constant, we can obviously see if it is a zero or not. |
--- |
| 5064 |
if (const ConstantFPSDNode *C = dyn_cast(Op)) |
0 |
5064 |
if (const ConstantFPSDNode *C = dyn_cast(Op)) |
0 |
| 5065 |
return !C->isZero(); |
0 |
5065 |
return !C->isZero(); |
0 |
| 5066 |
|
--- |
5066 |
|
--- |
| 5067 |
// Return false if we find any zero in a vector. |
--- |
5067 |
// Return false if we find any zero in a vector. |
--- |
| 5068 |
if (Op->getOpcode() == ISD::BUILD_VECTOR || |
0 |
5068 |
if (Op->getOpcode() == ISD::BUILD_VECTOR || |
0 |
| 5069 |
Op->getOpcode() == ISD::SPLAT_VECTOR) { |
0 |
5069 |
Op->getOpcode() == ISD::SPLAT_VECTOR) { |
0 |
| 5070 |
for (const SDValue &OpVal : Op->op_values()) { |
0 |
5070 |
for (const SDValue &OpVal : Op->op_values()) { |
0 |
| 5071 |
if (OpVal.isUndef()) |
0 |
5071 |
if (OpVal.isUndef()) |
0 |
| 5072 |
return false; |
0 |
5072 |
return false; |
0 |
| 5073 |
if (auto *C = dyn_cast(OpVal)) |
0 |
5073 |
if (auto *C = dyn_cast(OpVal)) |
0 |
| 5074 |
if (C->isZero()) |
0 |
5074 |
if (C->isZero()) |
0 |
| 5075 |
return false; |
0 |
5075 |
return false; |
0 |
| 5076 |
} |
--- |
5076 |
} |
--- |
| 5077 |
return true; |
0 |
5077 |
return true; |
0 |
| 5078 |
} |
--- |
5078 |
} |
--- |
| 5079 |
return false; |
0 |
5079 |
return false; |
0 |
| 5080 |
} |
--- |
5080 |
} |
--- |
| 5081 |
|
--- |
5081 |
|
--- |
| 5082 |
bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const { |
0 |
5082 |
bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const { |
0 |
| 5083 |
if (Depth >= MaxRecursionDepth) |
0 |
5083 |
if (Depth >= MaxRecursionDepth) |
0 |
| 5084 |
return false; // Limit search depth. |
0 |
5084 |
return false; // Limit search depth. |
0 |
| 5085 |
|
--- |
5085 |
|
--- |
| 5086 |
assert(!Op.getValueType().isFloatingPoint() && |
0 |
5086 |
assert(!Op.getValueType().isFloatingPoint() && |
0 |
| 5087 |
"Floating point types unsupported - use isKnownNeverZeroFloat"); |
--- |
5087 |
"Floating point types unsupported - use isKnownNeverZeroFloat"); |
--- |
| 5088 |
|
--- |
5088 |
|
--- |
| 5089 |
// If the value is a constant, we can obviously see if it is a zero or not. |
--- |
5089 |
// If the value is a constant, we can obviously see if it is a zero or not. |
--- |
| 5090 |
if (ISD::matchUnaryPredicate(Op, |
0 |
5090 |
if (ISD::matchUnaryPredicate(Op, |
0 |
| 5091 |
[](ConstantSDNode *C) { return !C->isZero(); })) |
0 |
5091 |
[](ConstantSDNode *C) { return !C->isZero(); })) |
0 |
| 5092 |
return true; |
0 |
5092 |
return true; |
0 |
| 5093 |
|
--- |
5093 |
|
--- |
| 5094 |
// TODO: Recognize more cases here. Most of the cases are also incomplete to |
--- |
5094 |
// TODO: Recognize more cases here. Most of the cases are also incomplete to |
--- |
| 5095 |
// some degree. |
--- |
5095 |
// some degree. |
--- |
| 5096 |
switch (Op.getOpcode()) { |
0 |
5096 |
switch (Op.getOpcode()) { |
0 |
| 5097 |
default: |
0 |
5097 |
default: |
0 |
| 5098 |
break; |
0 |
5098 |
break; |
0 |
| 5099 |
|
--- |
5099 |
|
--- |
| 5100 |
case ISD::OR: |
0 |
5100 |
case ISD::OR: |
0 |
| 5101 |
return isKnownNeverZero(Op.getOperand(1), Depth + 1) || |
0 |
5101 |
return isKnownNeverZero(Op.getOperand(1), Depth + 1) || |
0 |
| 5102 |
isKnownNeverZero(Op.getOperand(0), Depth + 1); |
0 |
5102 |
isKnownNeverZero(Op.getOperand(0), Depth + 1); |
0 |
| 5103 |
|
--- |
5103 |
|
--- |
| 5104 |
case ISD::VSELECT: |
0 |
5104 |
case ISD::VSELECT: |
0 |
| 5105 |
case ISD::SELECT: |
--- |
5105 |
case ISD::SELECT: |
--- |
| 5106 |
return isKnownNeverZero(Op.getOperand(1), Depth + 1) && |
0 |
5106 |
return isKnownNeverZero(Op.getOperand(1), Depth + 1) && |
0 |
| 5107 |
isKnownNeverZero(Op.getOperand(2), Depth + 1); |
0 |
5107 |
isKnownNeverZero(Op.getOperand(2), Depth + 1); |
0 |
| 5108 |
|
--- |
5108 |
|
--- |
| 5109 |
case ISD::SHL: |
0 |
5109 |
case ISD::SHL: |
0 |
| 5110 |
if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap()) |
0 |
5110 |
if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap()) |
0 |
| 5111 |
return isKnownNeverZero(Op.getOperand(0), Depth + 1); |
0 |
5111 |
return isKnownNeverZero(Op.getOperand(0), Depth + 1); |
0 |
| 5112 |
|
--- |
5112 |
|
--- |
| 5113 |
// 1 << X is never zero. TODO: This can be expanded if we can bound X. |
--- |
5113 |
// 1 << X is never zero. TODO: This can be expanded if we can bound X. |
--- |
| 5114 |
// The expression is really !Known.One[BitWidth-MaxLog2(Known):0].isZero() |
--- |
5114 |
// The expression is really !Known.One[BitWidth-MaxLog2(Known):0].isZero() |
--- |
| 5115 |
if (computeKnownBits(Op.getOperand(0), Depth + 1).One[0]) |
0 |
5115 |
if (computeKnownBits(Op.getOperand(0), Depth + 1).One[0]) |
0 |
| 5116 |
return true; |
0 |
5116 |
return true; |
0 |
| 5117 |
break; |
0 |
5117 |
break; |
0 |
| 5118 |
|
--- |
5118 |
|
--- |
| 5119 |
case ISD::UADDSAT: |
0 |
5119 |
case ISD::UADDSAT: |
0 |
| 5120 |
case ISD::UMAX: |
--- |
5120 |
case ISD::UMAX: |
--- |
| 5121 |
return isKnownNeverZero(Op.getOperand(1), Depth + 1) || |
0 |
5121 |
return isKnownNeverZero(Op.getOperand(1), Depth + 1) || |
0 |
| 5122 |
isKnownNeverZero(Op.getOperand(0), Depth + 1); |
0 |
5122 |
isKnownNeverZero(Op.getOperand(0), Depth + 1); |
0 |
| 5123 |
|
--- |
5123 |
|
--- |
| 5124 |
case ISD::UMIN: |
0 |
5124 |
case ISD::UMIN: |
0 |
| 5125 |
return isKnownNeverZero(Op.getOperand(1), Depth + 1) && |
0 |
5125 |
return isKnownNeverZero(Op.getOperand(1), Depth + 1) && |
0 |
| 5126 |
isKnownNeverZero(Op.getOperand(0), Depth + 1); |
0 |
5126 |
isKnownNeverZero(Op.getOperand(0), Depth + 1); |
0 |
| 5127 |
|
--- |
5127 |
|
--- |
| 5128 |
case ISD::ROTL: |
0 |
5128 |
case ISD::ROTL: |
0 |
| 5129 |
case ISD::ROTR: |
--- |
5129 |
case ISD::ROTR: |
--- |
| 5130 |
case ISD::BITREVERSE: |
--- |
5130 |
case ISD::BITREVERSE: |
--- |
| 5131 |
case ISD::BSWAP: |
--- |
5131 |
case ISD::BSWAP: |
--- |
| 5132 |
case ISD::CTPOP: |
--- |
5132 |
case ISD::CTPOP: |
--- |
| 5133 |
case ISD::ABS: |
--- |
5133 |
case ISD::ABS: |
--- |
| 5134 |
return isKnownNeverZero(Op.getOperand(0), Depth + 1); |
0 |
5134 |
return isKnownNeverZero(Op.getOperand(0), Depth + 1); |
0 |
| 5135 |
|
--- |
5135 |
|
--- |
| 5136 |
case ISD::SRA: |
0 |
5136 |
case ISD::SRA: |
0 |
| 5137 |
case ISD::SRL: |
--- |
5137 |
case ISD::SRL: |
--- |
| 5138 |
if (Op->getFlags().hasExact()) |
0 |
5138 |
if (Op->getFlags().hasExact()) |
0 |
| 5139 |
return isKnownNeverZero(Op.getOperand(0), Depth + 1); |
0 |
5139 |
return isKnownNeverZero(Op.getOperand(0), Depth + 1); |
0 |
| 5140 |
// Signed >> X is never zero. TODO: This can be expanded if we can bound X. |
--- |
5140 |
// Signed >> X is never zero. TODO: This can be expanded if we can bound X. |
--- |
| 5141 |
// The expression is really |
--- |
5141 |
// The expression is really |
--- |
| 5142 |
// !Known.One[SignBit:SignBit-(BitWidth-MaxLog2(Known))].isZero() |
--- |
5142 |
// !Known.One[SignBit:SignBit-(BitWidth-MaxLog2(Known))].isZero() |
--- |
| 5143 |
if (computeKnownBits(Op.getOperand(0), Depth + 1).isNegative()) |
0 |
5143 |
if (computeKnownBits(Op.getOperand(0), Depth + 1).isNegative()) |
0 |
| 5144 |
return true; |
0 |
5144 |
return true; |
0 |
| 5145 |
break; |
0 |
5145 |
break; |
0 |
| 5146 |
|
--- |
5146 |
|
--- |
| 5147 |
case ISD::UDIV: |
0 |
5147 |
case ISD::UDIV: |
0 |
| 5148 |
case ISD::SDIV: |
--- |
5148 |
case ISD::SDIV: |
--- |
| 5149 |
// div exact can only produce a zero if the dividend is zero. |
--- |
5149 |
// div exact can only produce a zero if the dividend is zero. |
--- |
| 5150 |
// TODO: For udiv this is also true if Op1 u<= Op0 |
--- |
5150 |
// TODO: For udiv this is also true if Op1 u<= Op0 |
--- |
| 5151 |
if (Op->getFlags().hasExact()) |
0 |
5151 |
if (Op->getFlags().hasExact()) |
0 |
| 5152 |
return isKnownNeverZero(Op.getOperand(0), Depth + 1); |
0 |
5152 |
return isKnownNeverZero(Op.getOperand(0), Depth + 1); |
0 |
| 5153 |
break; |
0 |
5153 |
break; |
0 |
| 5154 |
|
--- |
5154 |
|
--- |
| 5155 |
case ISD::ADD: |
0 |
5155 |
case ISD::ADD: |
0 |
| 5156 |
if (Op->getFlags().hasNoUnsignedWrap()) |
0 |
5156 |
if (Op->getFlags().hasNoUnsignedWrap()) |
0 |
| 5157 |
if (isKnownNeverZero(Op.getOperand(1), Depth + 1) || |
0 |
5157 |
if (isKnownNeverZero(Op.getOperand(1), Depth + 1) || |
0 |
| 5158 |
isKnownNeverZero(Op.getOperand(0), Depth + 1)) |
0 |
5158 |
isKnownNeverZero(Op.getOperand(0), Depth + 1)) |
0 |
| 5159 |
return true; |
0 |
5159 |
return true; |
0 |
| 5160 |
// TODO: There are a lot more cases we can prove for add. |
--- |
5160 |
// TODO: There are a lot more cases we can prove for add. |
--- |
| 5161 |
break; |
0 |
5161 |
break; |
0 |
| 5162 |
|
--- |
5162 |
|
--- |
| 5163 |
case ISD::SUB: { |
0 |
5163 |
case ISD::SUB: { |
0 |
| 5164 |
if (isNullConstant(Op.getOperand(0))) |
0 |
5164 |
if (isNullConstant(Op.getOperand(0))) |
0 |
| 5165 |
return isKnownNeverZero(Op.getOperand(1), Depth + 1); |
0 |
5165 |
return isKnownNeverZero(Op.getOperand(1), Depth + 1); |
0 |
| 5166 |
|
--- |
5166 |
|
--- |
| 5167 |
std::optional ne = |
--- |
5167 |
std::optional ne = |
--- |
| 5168 |
KnownBits::ne(computeKnownBits(Op.getOperand(0), Depth + 1), |
0 |
5168 |
KnownBits::ne(computeKnownBits(Op.getOperand(0), Depth + 1), |
0 |
| 5169 |
computeKnownBits(Op.getOperand(1), Depth + 1)); |
0 |
5169 |
computeKnownBits(Op.getOperand(1), Depth + 1)); |
0 |
| 5170 |
return ne && *ne; |
0 |
5170 |
return ne && *ne; |
0 |
| 5171 |
} |
--- |
5171 |
} |
--- |
| 5172 |
|
--- |
5172 |
|
--- |
| 5173 |
case ISD::MUL: |
0 |
5173 |
case ISD::MUL: |
0 |
| 5174 |
if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap()) |
0 |
5174 |
if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap()) |
0 |
| 5175 |
if (isKnownNeverZero(Op.getOperand(1), Depth + 1) && |
0 |
5175 |
if (isKnownNeverZero(Op.getOperand(1), Depth + 1) && |
0 |
| 5176 |
isKnownNeverZero(Op.getOperand(0), Depth + 1)) |
0 |
5176 |
isKnownNeverZero(Op.getOperand(0), Depth + 1)) |
0 |
| 5177 |
return true; |
0 |
5177 |
return true; |
0 |
| 5178 |
break; |
0 |
5178 |
break; |
0 |
| 5179 |
|
--- |
5179 |
|
--- |
| 5180 |
case ISD::ZERO_EXTEND: |
0 |
5180 |
case ISD::ZERO_EXTEND: |
0 |
| 5181 |
case ISD::SIGN_EXTEND: |
--- |
5181 |
case ISD::SIGN_EXTEND: |
--- |
| 5182 |
return isKnownNeverZero(Op.getOperand(0), Depth + 1); |
0 |
5182 |
return isKnownNeverZero(Op.getOperand(0), Depth + 1); |
0 |
| 5183 |
} |
--- |
5183 |
} |
--- |
| 5184 |
|
--- |
5184 |
|
--- |
| 5185 |
return computeKnownBits(Op, Depth).isNonZero(); |
0 |
5185 |
return computeKnownBits(Op, Depth).isNonZero(); |
0 |
| 5186 |
} |
--- |
5186 |
} |
--- |
| 5187 |
|
--- |
5187 |
|
--- |
| 5188 |
bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { |
0 |
5188 |
bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { |
0 |
| 5189 |
// Check the obvious case. |
--- |
5189 |
// Check the obvious case. |
--- |
| 5190 |
if (A == B) return true; |
0 |
5190 |
if (A == B) return true; |
0 |
| 5191 |
|
--- |
5191 |
|
--- |
| 5192 |
// For negative and positive zero. |
--- |
5192 |
// For negative and positive zero. |
--- |
| 5193 |
if (const ConstantFPSDNode *CA = dyn_cast(A)) |
0 |
5193 |
if (const ConstantFPSDNode *CA = dyn_cast(A)) |
0 |
| 5194 |
if (const ConstantFPSDNode *CB = dyn_cast(B)) |
0 |
5194 |
if (const ConstantFPSDNode *CB = dyn_cast(B)) |
0 |
| 5195 |
if (CA->isZero() && CB->isZero()) return true; |
0 |
5195 |
if (CA->isZero() && CB->isZero()) return true; |
0 |
| 5196 |
|
--- |
5196 |
|
--- |
| 5197 |
// Otherwise they may not be equal. |
--- |
5197 |
// Otherwise they may not be equal. |
--- |
| 5198 |
return false; |
0 |
5198 |
return false; |
0 |
| 5199 |
} |
--- |
5199 |
} |
--- |
| 5200 |
|
--- |
5200 |
|
--- |
| 5201 |
// Only bits set in Mask must be negated, other bits may be arbitrary. |
--- |
5201 |
// Only bits set in Mask must be negated, other bits may be arbitrary. |
--- |
| 5202 |
SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) { |
0 |
5202 |
SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) { |
0 |
| 5203 |
if (isBitwiseNot(V, AllowUndefs)) |
0 |
5203 |
if (isBitwiseNot(V, AllowUndefs)) |
0 |
| 5204 |
return V.getOperand(0); |
0 |
5204 |
return V.getOperand(0); |
0 |
| 5205 |
|
--- |
5205 |
|
--- |
| 5206 |
// Handle any_extend (not (truncate X)) pattern, where Mask only sets |
--- |
5206 |
// Handle any_extend (not (truncate X)) pattern, where Mask only sets |
--- |
| 5207 |
// bits in the non-extended part. |
--- |
5207 |
// bits in the non-extended part. |
--- |
| 5208 |
ConstantSDNode *MaskC = isConstOrConstSplat(Mask); |
0 |
5208 |
ConstantSDNode *MaskC = isConstOrConstSplat(Mask); |
0 |
| 5209 |
if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND) |
0 |
5209 |
if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND) |
0 |
| 5210 |
return SDValue(); |
0 |
5210 |
return SDValue(); |
0 |
| 5211 |
SDValue ExtArg = V.getOperand(0); |
0 |
5211 |
SDValue ExtArg = V.getOperand(0); |
0 |
| 5212 |
if (ExtArg.getScalarValueSizeInBits() >= |
0 |
5212 |
if (ExtArg.getScalarValueSizeInBits() >= |
0 |
| 5213 |
MaskC->getAPIntValue().getActiveBits() && |
0 |
5213 |
MaskC->getAPIntValue().getActiveBits() && |
0 |
| 5214 |
isBitwiseNot(ExtArg, AllowUndefs) && |
0 |
5214 |
isBitwiseNot(ExtArg, AllowUndefs) && |
0 |
| 5215 |
ExtArg.getOperand(0).getOpcode() == ISD::TRUNCATE && |
0 |
5215 |
ExtArg.getOperand(0).getOpcode() == ISD::TRUNCATE && |
0 |
| 5216 |
ExtArg.getOperand(0).getOperand(0).getValueType() == V.getValueType()) |
0 |
5216 |
ExtArg.getOperand(0).getOperand(0).getValueType() == V.getValueType()) |
0 |
| 5217 |
return ExtArg.getOperand(0).getOperand(0); |
0 |
5217 |
return ExtArg.getOperand(0).getOperand(0); |
0 |
| 5218 |
return SDValue(); |
0 |
5218 |
return SDValue(); |
0 |
| 5219 |
} |
--- |
5219 |
} |
--- |
| 5220 |
|
--- |
5220 |
|
--- |
| 5221 |
static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) { |
2 |
5221 |
static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) { |
2 |
| 5222 |
// Match masked merge pattern (X & ~M) op (Y & M) |
--- |
5222 |
// Match masked merge pattern (X & ~M) op (Y & M) |
--- |
| 5223 |
// Including degenerate case (X & ~M) op M |
--- |
5223 |
// Including degenerate case (X & ~M) op M |
--- |
| 5224 |
auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask, |
0 |
5224 |
auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask, |
0 |
| 5225 |
SDValue Other) { |
--- |
5225 |
SDValue Other) { |
--- |
| 5226 |
if (SDValue NotOperand = |
0 |
5226 |
if (SDValue NotOperand = |
0 |
| 5227 |
getBitwiseNotOperand(Not, Mask, /* AllowUndefs */ true)) { |
0 |
5227 |
getBitwiseNotOperand(Not, Mask, /* AllowUndefs */ true)) { |
0 |
| 5228 |
if (NotOperand->getOpcode() == ISD::ZERO_EXTEND || |
0 |
5228 |
if (NotOperand->getOpcode() == ISD::ZERO_EXTEND || |
0 |
| 5229 |
NotOperand->getOpcode() == ISD::TRUNCATE) |
0 |
5229 |
NotOperand->getOpcode() == ISD::TRUNCATE) |
0 |
| 5230 |
NotOperand = NotOperand->getOperand(0); |
0 |
5230 |
NotOperand = NotOperand->getOperand(0); |
0 |
| 5231 |
|
--- |
5231 |
|
--- |
| 5232 |
if (Other == NotOperand) |
0 |
5232 |
if (Other == NotOperand) |
0 |
| 5233 |
return true; |
0 |
5233 |
return true; |
0 |
| 5234 |
if (Other->getOpcode() == ISD::AND) |
0 |
5234 |
if (Other->getOpcode() == ISD::AND) |
0 |
| 5235 |
return NotOperand == Other->getOperand(0) || |
0 |
5235 |
return NotOperand == Other->getOperand(0) || |
0 |
| 5236 |
NotOperand == Other->getOperand(1); |
0 |
5236 |
NotOperand == Other->getOperand(1); |
0 |
| 5237 |
} |
--- |
5237 |
} |
--- |
| 5238 |
return false; |
0 |
5238 |
return false; |
0 |
| 5239 |
}; |
--- |
5239 |
}; |
--- |
| 5240 |
|
--- |
5240 |
|
--- |
| 5241 |
if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE) |
2 |
5241 |
if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE) |
2 |
| 5242 |
A = A->getOperand(0); |
0 |
5242 |
A = A->getOperand(0); |
0 |
| 5243 |
|
--- |
5243 |
|
--- |
| 5244 |
if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE) |
2 |
5244 |
if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE) |
2 |
| 5245 |
B = B->getOperand(0); |
0 |
5245 |
B = B->getOperand(0); |
0 |
| 5246 |
|
--- |
5246 |
|
--- |
| 5247 |
if (A->getOpcode() == ISD::AND) |
2 |
5247 |
if (A->getOpcode() == ISD::AND) |
2 |
| 5248 |
return MatchNoCommonBitsPattern(A->getOperand(0), A->getOperand(1), B) || |
0 |
5248 |
return MatchNoCommonBitsPattern(A->getOperand(0), A->getOperand(1), B) || |
0 |
| 5249 |
MatchNoCommonBitsPattern(A->getOperand(1), A->getOperand(0), B); |
0 |
5249 |
MatchNoCommonBitsPattern(A->getOperand(1), A->getOperand(0), B); |
0 |
| 5250 |
return false; |
2 |
5250 |
return false; |
2 |
| 5251 |
} |
--- |
5251 |
} |
--- |
| 5252 |
|
--- |
5252 |
|
--- |
| 5253 |
// FIXME: unify with llvm::haveNoCommonBitsSet. |
--- |
5253 |
// FIXME: unify with llvm::haveNoCommonBitsSet. |
--- |
| 5254 |
bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const { |
1 |
5254 |
bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const { |
1 |
| 5255 |
assert(A.getValueType() == B.getValueType() && |
1 |
5255 |
assert(A.getValueType() == B.getValueType() && |
1 |
| 5256 |
"Values must have the same type"); |
--- |
5256 |
"Values must have the same type"); |
--- |
| 5257 |
if (haveNoCommonBitsSetCommutative(A, B) || |
2 |
5257 |
if (haveNoCommonBitsSetCommutative(A, B) || |
2 |
| 5258 |
haveNoCommonBitsSetCommutative(B, A)) |
1 |
5258 |
haveNoCommonBitsSetCommutative(B, A)) |
1 |
| 5259 |
return true; |
0 |
5259 |
return true; |
0 |
| 5260 |
return KnownBits::haveNoCommonBitsSet(computeKnownBits(A), |
2 |
5260 |
return KnownBits::haveNoCommonBitsSet(computeKnownBits(A), |
2 |
| 5261 |
computeKnownBits(B)); |
3 |
5261 |
computeKnownBits(B)); |
3 |
| 5262 |
} |
--- |
5262 |
} |
--- |
| 5263 |
|
--- |
5263 |
|
--- |
| 5264 |
static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step, |
0 |
5264 |
static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step, |
0 |
| 5265 |
SelectionDAG &DAG) { |
--- |
5265 |
SelectionDAG &DAG) { |
--- |
| 5266 |
if (cast(Step)->isZero()) |
0 |
5266 |
if (cast(Step)->isZero()) |
0 |
| 5267 |
return DAG.getConstant(0, DL, VT); |
0 |
5267 |
return DAG.getConstant(0, DL, VT); |
0 |
| 5268 |
|
--- |
5268 |
|
--- |
| 5269 |
return SDValue(); |
0 |
5269 |
return SDValue(); |
0 |
| 5270 |
} |
--- |
5270 |
} |
--- |
| 5271 |
|
--- |
5271 |
|
--- |
| 5272 |
static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT, |
0 |
5272 |
static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT, |
0 |
| 5273 |
ArrayRef Ops, |
--- |
5273 |
ArrayRef Ops, |
--- |
| 5274 |
SelectionDAG &DAG) { |
--- |
5274 |
SelectionDAG &DAG) { |
--- |
| 5275 |
int NumOps = Ops.size(); |
0 |
5275 |
int NumOps = Ops.size(); |
0 |
| 5276 |
assert(NumOps != 0 && "Can't build an empty vector!"); |
0 |
5276 |
assert(NumOps != 0 && "Can't build an empty vector!"); |
0 |
| 5277 |
assert(!VT.isScalableVector() && |
0 |
5277 |
assert(!VT.isScalableVector() && |
0 |
| 5278 |
"BUILD_VECTOR cannot be used with scalable types"); |
--- |
5278 |
"BUILD_VECTOR cannot be used with scalable types"); |
--- |
| 5279 |
assert(VT.getVectorNumElements() == (unsigned)NumOps && |
0 |
5279 |
assert(VT.getVectorNumElements() == (unsigned)NumOps && |
0 |
| 5280 |
"Incorrect element count in BUILD_VECTOR!"); |
--- |
5280 |
"Incorrect element count in BUILD_VECTOR!"); |
--- |
| 5281 |
|
--- |
5281 |
|
--- |
| 5282 |
// BUILD_VECTOR of UNDEFs is UNDEF. |
--- |
5282 |
// BUILD_VECTOR of UNDEFs is UNDEF. |
--- |
| 5283 |
if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); })) |
0 |
5283 |
if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); })) |
0 |
| 5284 |
return DAG.getUNDEF(VT); |
0 |
5284 |
return DAG.getUNDEF(VT); |
0 |
| 5285 |
|
--- |
5285 |
|
--- |
| 5286 |
// BUILD_VECTOR of seq extract/insert from the same vector + type is Identity. |
--- |
5286 |
// BUILD_VECTOR of seq extract/insert from the same vector + type is Identity. |
--- |
| 5287 |
SDValue IdentitySrc; |
0 |
5287 |
SDValue IdentitySrc; |
0 |
| 5288 |
bool IsIdentity = true; |
0 |
5288 |
bool IsIdentity = true; |
0 |
| 5289 |
for (int i = 0; i != NumOps; ++i) { |
0 |
5289 |
for (int i = 0; i != NumOps; ++i) { |
0 |
| 5290 |
if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT || |
0 |
5290 |
if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT || |
0 |
| 5291 |
Ops[i].getOperand(0).getValueType() != VT || |
0 |
5291 |
Ops[i].getOperand(0).getValueType() != VT || |
0 |
| 5292 |
(IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) || |
0 |
5292 |
(IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) || |
0 |
| 5293 |
!isa(Ops[i].getOperand(1)) || |
0 |
5293 |
!isa(Ops[i].getOperand(1)) || |
0 |
| 5294 |
cast(Ops[i].getOperand(1))->getAPIntValue() != i) { |
0 |
5294 |
cast(Ops[i].getOperand(1))->getAPIntValue() != i) { |
0 |
| 5295 |
IsIdentity = false; |
0 |
5295 |
IsIdentity = false; |
0 |
| 5296 |
break; |
0 |
5296 |
break; |
0 |
| 5297 |
} |
--- |
5297 |
} |
--- |
| 5298 |
IdentitySrc = Ops[i].getOperand(0); |
0 |
5298 |
IdentitySrc = Ops[i].getOperand(0); |
0 |
| 5299 |
} |
--- |
5299 |
} |
--- |
| 5300 |
if (IsIdentity) |
0 |
5300 |
if (IsIdentity) |
0 |
| 5301 |
return IdentitySrc; |
0 |
5301 |
return IdentitySrc; |
0 |
| 5302 |
|
--- |
5302 |
|
--- |
| 5303 |
return SDValue(); |
0 |
5303 |
return SDValue(); |
0 |
| 5304 |
} |
--- |
5304 |
} |
--- |
| 5305 |
|
--- |
5305 |
|
--- |
| 5306 |
/// Try to simplify vector concatenation to an input value, undef, or build |
--- |
5306 |
/// Try to simplify vector concatenation to an input value, undef, or build |
--- |
| 5307 |
/// vector. |
--- |
5307 |
/// vector. |
--- |
| 5308 |
static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT, |
0 |
5308 |
static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT, |
0 |
| 5309 |
ArrayRef Ops, |
--- |
5309 |
ArrayRef Ops, |
--- |
| 5310 |
SelectionDAG &DAG) { |
--- |
5310 |
SelectionDAG &DAG) { |
--- |
| 5311 |
assert(!Ops.empty() && "Can't concatenate an empty list of vectors!"); |
0 |
5311 |
assert(!Ops.empty() && "Can't concatenate an empty list of vectors!"); |
0 |
| 5312 |
assert(llvm::all_of(Ops, |
0 |
5312 |
assert(llvm::all_of(Ops, |
0 |
| 5313 |
[Ops](SDValue Op) { |
--- |
5313 |
[Ops](SDValue Op) { |
--- |
| 5314 |
return Ops[0].getValueType() == Op.getValueType(); |
--- |
5314 |
return Ops[0].getValueType() == Op.getValueType(); |
--- |
| 5315 |
}) && |
--- |
5315 |
}) && |
--- |
| 5316 |
"Concatenation of vectors with inconsistent value types!"); |
--- |
5316 |
"Concatenation of vectors with inconsistent value types!"); |
--- |
| 5317 |
assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) == |
0 |
5317 |
assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) == |
0 |
| 5318 |
VT.getVectorElementCount() && |
--- |
5318 |
VT.getVectorElementCount() && |
--- |
| 5319 |
"Incorrect element count in vector concatenation!"); |
--- |
5319 |
"Incorrect element count in vector concatenation!"); |
--- |
| 5320 |
|
--- |
5320 |
|
--- |
| 5321 |
if (Ops.size() == 1) |
0 |
5321 |
if (Ops.size() == 1) |
0 |
| 5322 |
return Ops[0]; |
0 |
5322 |
return Ops[0]; |
0 |
| 5323 |
|
--- |
5323 |
|
--- |
| 5324 |
// Concat of UNDEFs is UNDEF. |
--- |
5324 |
// Concat of UNDEFs is UNDEF. |
--- |
| 5325 |
if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); })) |
0 |
5325 |
if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); })) |
0 |
| 5326 |
return DAG.getUNDEF(VT); |
0 |
5326 |
return DAG.getUNDEF(VT); |
0 |
| 5327 |
|
--- |
5327 |
|
--- |
| 5328 |
// Scan the operands and look for extract operations from a single source |
--- |
5328 |
// Scan the operands and look for extract operations from a single source |
--- |
| 5329 |
// that correspond to insertion at the same location via this concatenation: |
--- |
5329 |
// that correspond to insertion at the same location via this concatenation: |
--- |
| 5330 |
// concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ... |
--- |
5330 |
// concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ... |
--- |
| 5331 |
SDValue IdentitySrc; |
0 |
5331 |
SDValue IdentitySrc; |
0 |
| 5332 |
bool IsIdentity = true; |
0 |
5332 |
bool IsIdentity = true; |
0 |
| 5333 |
for (unsigned i = 0, e = Ops.size(); i != e; ++i) { |
0 |
5333 |
for (unsigned i = 0, e = Ops.size(); i != e; ++i) { |
0 |
| 5334 |
SDValue Op = Ops[i]; |
0 |
5334 |
SDValue Op = Ops[i]; |
0 |
| 5335 |
unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements(); |
0 |
5335 |
unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements(); |
0 |
| 5336 |
if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR || |
0 |
5336 |
if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR || |
0 |
| 5337 |
Op.getOperand(0).getValueType() != VT || |
0 |
5337 |
Op.getOperand(0).getValueType() != VT || |
0 |
| 5338 |
(IdentitySrc && Op.getOperand(0) != IdentitySrc) || |
0 |
5338 |
(IdentitySrc && Op.getOperand(0) != IdentitySrc) || |
0 |
| 5339 |
Op.getConstantOperandVal(1) != IdentityIndex) { |
0 |
5339 |
Op.getConstantOperandVal(1) != IdentityIndex) { |
0 |
| 5340 |
IsIdentity = false; |
0 |
5340 |
IsIdentity = false; |
0 |
| 5341 |
break; |
0 |
5341 |
break; |
0 |
| 5342 |
} |
--- |
5342 |
} |
--- |
| 5343 |
assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) && |
0 |
5343 |
assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) && |
0 |
| 5344 |
"Unexpected identity source vector for concat of extracts"); |
--- |
5344 |
"Unexpected identity source vector for concat of extracts"); |
--- |
| 5345 |
IdentitySrc = Op.getOperand(0); |
0 |
5345 |
IdentitySrc = Op.getOperand(0); |
0 |
| 5346 |
} |
--- |
5346 |
} |
--- |
| 5347 |
if (IsIdentity) { |
0 |
5347 |
if (IsIdentity) { |
0 |
| 5348 |
assert(IdentitySrc && "Failed to set source vector of extracts"); |
0 |
5348 |
assert(IdentitySrc && "Failed to set source vector of extracts"); |
0 |
| 5349 |
return IdentitySrc; |
0 |
5349 |
return IdentitySrc; |
0 |
| 5350 |
} |
--- |
5350 |
} |
--- |
| 5351 |
|
--- |
5351 |
|
--- |
| 5352 |
// The code below this point is only designed to work for fixed width |
--- |
5352 |
// The code below this point is only designed to work for fixed width |
--- |
| 5353 |
// vectors, so we bail out for now. |
--- |
5353 |
// vectors, so we bail out for now. |
--- |
| 5354 |
if (VT.isScalableVector()) |
0 |
5354 |
if (VT.isScalableVector()) |
0 |
| 5355 |
return SDValue(); |
0 |
5355 |
return SDValue(); |
0 |
| 5356 |
|
--- |
5356 |
|
--- |
| 5357 |
// A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be |
--- |
5357 |
// A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be |
--- |
| 5358 |
// simplified to one big BUILD_VECTOR. |
--- |
5358 |
// simplified to one big BUILD_VECTOR. |
--- |
| 5359 |
// FIXME: Add support for SCALAR_TO_VECTOR as well. |
--- |
5359 |
// FIXME: Add support for SCALAR_TO_VECTOR as well. |
--- |
| 5360 |
EVT SVT = VT.getScalarType(); |
0 |
5360 |
EVT SVT = VT.getScalarType(); |
0 |
| 5361 |
SmallVector Elts; |
0 |
5361 |
SmallVector Elts; |
0 |
| 5362 |
for (SDValue Op : Ops) { |
0 |
5362 |
for (SDValue Op : Ops) { |
0 |
| 5363 |
EVT OpVT = Op.getValueType(); |
0 |
5363 |
EVT OpVT = Op.getValueType(); |
0 |
| 5364 |
if (Op.isUndef()) |
0 |
5364 |
if (Op.isUndef()) |
0 |
| 5365 |
Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT)); |
0 |
5365 |
Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT)); |
0 |
| 5366 |
else if (Op.getOpcode() == ISD::BUILD_VECTOR) |
0 |
5366 |
else if (Op.getOpcode() == ISD::BUILD_VECTOR) |
0 |
| 5367 |
Elts.append(Op->op_begin(), Op->op_end()); |
0 |
5367 |
Elts.append(Op->op_begin(), Op->op_end()); |
0 |
| 5368 |
else |
--- |
5368 |
else |
--- |
| 5369 |
return SDValue(); |
0 |
5369 |
return SDValue(); |
0 |
| 5370 |
} |
--- |
5370 |
} |
--- |
| 5371 |
|
--- |
5371 |
|
--- |
| 5372 |
// BUILD_VECTOR requires all inputs to be of the same type, find the |
--- |
5372 |
// BUILD_VECTOR requires all inputs to be of the same type, find the |
--- |
| 5373 |
// maximum type and extend them all. |
--- |
5373 |
// maximum type and extend them all. |
--- |
| 5374 |
for (SDValue Op : Elts) |
0 |
5374 |
for (SDValue Op : Elts) |
0 |
| 5375 |
SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT); |
0 |
5375 |
SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT); |
0 |
| 5376 |
|
--- |
5376 |
|
--- |
| 5377 |
if (SVT.bitsGT(VT.getScalarType())) { |
0 |
5377 |
if (SVT.bitsGT(VT.getScalarType())) { |
0 |
| 5378 |
for (SDValue &Op : Elts) { |
0 |
5378 |
for (SDValue &Op : Elts) { |
0 |
| 5379 |
if (Op.isUndef()) |
0 |
5379 |
if (Op.isUndef()) |
0 |
| 5380 |
Op = DAG.getUNDEF(SVT); |
0 |
5380 |
Op = DAG.getUNDEF(SVT); |
0 |
| 5381 |
else |
--- |
5381 |
else |
--- |
| 5382 |
Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT) |
0 |
5382 |
Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT) |
0 |
| 5383 |
? DAG.getZExtOrTrunc(Op, DL, SVT) |
0 |
5383 |
? DAG.getZExtOrTrunc(Op, DL, SVT) |
0 |
| 5384 |
: DAG.getSExtOrTrunc(Op, DL, SVT); |
0 |
5384 |
: DAG.getSExtOrTrunc(Op, DL, SVT); |
0 |
| 5385 |
} |
--- |
5385 |
} |
--- |
| 5386 |
} |
--- |
5386 |
} |
--- |
| 5387 |
|
--- |
5387 |
|
--- |
| 5388 |
SDValue V = DAG.getBuildVector(VT, DL, Elts); |
0 |
5388 |
SDValue V = DAG.getBuildVector(VT, DL, Elts); |
0 |
| 5389 |
NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG); |
0 |
5389 |
NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG); |
0 |
| 5390 |
return V; |
0 |
5390 |
return V; |
0 |
| 5391 |
} |
0 |
5391 |
} |
0 |
| 5392 |
|
--- |
5392 |
|
--- |
| 5393 |
/// Gets or creates the specified node. |
--- |
5393 |
/// Gets or creates the specified node. |
--- |
| 5394 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) { |
9 |
5394 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) { |
9 |
| 5395 |
FoldingSetNodeID ID; |
9 |
5395 |
FoldingSetNodeID ID; |
9 |
| 5396 |
AddNodeIDNode(ID, Opcode, getVTList(VT), std::nullopt); |
9 |
5396 |
AddNodeIDNode(ID, Opcode, getVTList(VT), std::nullopt); |
9 |
| 5397 |
void *IP = nullptr; |
9 |
5397 |
void *IP = nullptr; |
9 |
| 5398 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) |
9 |
5398 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) |
9 |
| 5399 |
return SDValue(E, 0); |
5 |
5399 |
return SDValue(E, 0); |
5 |
| 5400 |
|
--- |
5400 |
|
--- |
| 5401 |
auto *N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), |
4 |
5401 |
auto *N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), |
4 |
| 5402 |
getVTList(VT)); |
4 |
5402 |
getVTList(VT)); |
4 |
| 5403 |
CSEMap.InsertNode(N, IP); |
4 |
5403 |
CSEMap.InsertNode(N, IP); |
4 |
| 5404 |
|
--- |
5404 |
|
--- |
| 5405 |
InsertNode(N); |
4 |
5405 |
InsertNode(N); |
4 |
| 5406 |
SDValue V = SDValue(N, 0); |
4 |
5406 |
SDValue V = SDValue(N, 0); |
4 |
| 5407 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
4 |
5407 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
4 |
| 5408 |
return V; |
4 |
5408 |
return V; |
4 |
| 5409 |
} |
9 |
5409 |
} |
9 |
| 5410 |
|
--- |
5410 |
|
--- |
| 5411 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
3 |
5411 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
3 |
| 5412 |
SDValue N1) { |
--- |
5412 |
SDValue N1) { |
--- |
| 5413 |
SDNodeFlags Flags; |
3 |
5413 |
SDNodeFlags Flags; |
3 |
| 5414 |
if (Inserter) |
3 |
5414 |
if (Inserter) |
3 |
| 5415 |
Flags = Inserter->getFlags(); |
0 |
5415 |
Flags = Inserter->getFlags(); |
0 |
| 5416 |
return getNode(Opcode, DL, VT, N1, Flags); |
3 |
5416 |
return getNode(Opcode, DL, VT, N1, Flags); |
3 |
| 5417 |
} |
--- |
5417 |
} |
--- |
| 5418 |
|
--- |
5418 |
|
--- |
| 5419 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
17 |
5419 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
17 |
| 5420 |
SDValue N1, const SDNodeFlags Flags) { |
--- |
5420 |
SDValue N1, const SDNodeFlags Flags) { |
--- |
| 5421 |
assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!"); |
17 |
5421 |
assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!"); |
17 |
| 5422 |
// Constant fold unary operations with an integer constant operand. Even |
--- |
5422 |
// Constant fold unary operations with an integer constant operand. Even |
--- |
| 5423 |
// opaque constant will be folded, because the folding of unary operations |
--- |
5423 |
// opaque constant will be folded, because the folding of unary operations |
--- |
| 5424 |
// doesn't create new constants with different values. Nevertheless, the |
--- |
5424 |
// doesn't create new constants with different values. Nevertheless, the |
--- |
| 5425 |
// opaque flag is preserved during folding to prevent future folding with |
--- |
5425 |
// opaque flag is preserved during folding to prevent future folding with |
--- |
| 5426 |
// other constants. |
--- |
5426 |
// other constants. |
--- |
| 5427 |
if (ConstantSDNode *C = dyn_cast(N1)) { |
17 |
5427 |
if (ConstantSDNode *C = dyn_cast(N1)) { |
17 |
| 5428 |
const APInt &Val = C->getAPIntValue(); |
0 |
5428 |
const APInt &Val = C->getAPIntValue(); |
0 |
| 5429 |
switch (Opcode) { |
0 |
5429 |
switch (Opcode) { |
0 |
| 5430 |
default: break; |
0 |
5430 |
default: break; |
0 |
| 5431 |
case ISD::SIGN_EXTEND: |
0 |
5431 |
case ISD::SIGN_EXTEND: |
0 |
| 5432 |
return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT, |
0 |
5432 |
return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT, |
0 |
| 5433 |
C->isTargetOpcode(), C->isOpaque()); |
0 |
5433 |
C->isTargetOpcode(), C->isOpaque()); |
0 |
| 5434 |
case ISD::TRUNCATE: |
0 |
5434 |
case ISD::TRUNCATE: |
0 |
| 5435 |
if (C->isOpaque()) |
0 |
5435 |
if (C->isOpaque()) |
0 |
| 5436 |
break; |
0 |
5436 |
break; |
0 |
| 5437 |
[[fallthrough]]; |
--- |
5437 |
[[fallthrough]]; |
--- |
| 5438 |
case ISD::ZERO_EXTEND: |
--- |
5438 |
case ISD::ZERO_EXTEND: |
--- |
| 5439 |
return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT, |
0 |
5439 |
return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT, |
0 |
| 5440 |
C->isTargetOpcode(), C->isOpaque()); |
0 |
5440 |
C->isTargetOpcode(), C->isOpaque()); |
0 |
| 5441 |
case ISD::ANY_EXTEND: |
0 |
5441 |
case ISD::ANY_EXTEND: |
0 |
| 5442 |
// Some targets like RISCV prefer to sign extend some types. |
--- |
5442 |
// Some targets like RISCV prefer to sign extend some types. |
--- |
| 5443 |
if (TLI->isSExtCheaperThanZExt(N1.getValueType(), VT)) |
0 |
5443 |
if (TLI->isSExtCheaperThanZExt(N1.getValueType(), VT)) |
0 |
| 5444 |
return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT, |
0 |
5444 |
return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT, |
0 |
| 5445 |
C->isTargetOpcode(), C->isOpaque()); |
0 |
5445 |
C->isTargetOpcode(), C->isOpaque()); |
0 |
| 5446 |
return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT, |
0 |
5446 |
return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT, |
0 |
| 5447 |
C->isTargetOpcode(), C->isOpaque()); |
0 |
5447 |
C->isTargetOpcode(), C->isOpaque()); |
0 |
| 5448 |
case ISD::UINT_TO_FP: |
0 |
5448 |
case ISD::UINT_TO_FP: |
0 |
| 5449 |
case ISD::SINT_TO_FP: { |
--- |
5449 |
case ISD::SINT_TO_FP: { |
--- |
| 5450 |
APFloat apf(EVTToAPFloatSemantics(VT), |
--- |
5450 |
APFloat apf(EVTToAPFloatSemantics(VT), |
--- |
| 5451 |
APInt::getZero(VT.getSizeInBits())); |
0 |
5451 |
APInt::getZero(VT.getSizeInBits())); |
0 |
| 5452 |
(void)apf.convertFromAPInt(Val, |
0 |
5452 |
(void)apf.convertFromAPInt(Val, |
0 |
| 5453 |
Opcode==ISD::SINT_TO_FP, |
--- |
5453 |
Opcode==ISD::SINT_TO_FP, |
--- |
| 5454 |
APFloat::rmNearestTiesToEven); |
--- |
5454 |
APFloat::rmNearestTiesToEven); |
--- |
| 5455 |
return getConstantFP(apf, DL, VT); |
0 |
5455 |
return getConstantFP(apf, DL, VT); |
0 |
| 5456 |
} |
0 |
5456 |
} |
0 |
| 5457 |
case ISD::BITCAST: |
0 |
5457 |
case ISD::BITCAST: |
0 |
| 5458 |
if (VT == MVT::f16 && C->getValueType(0) == MVT::i16) |
0 |
5458 |
if (VT == MVT::f16 && C->getValueType(0) == MVT::i16) |
0 |
| 5459 |
return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT); |
0 |
5459 |
return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT); |
0 |
| 5460 |
if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) |
0 |
5460 |
if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) |
0 |
| 5461 |
return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT); |
0 |
5461 |
return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT); |
0 |
| 5462 |
if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) |
0 |
5462 |
if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) |
0 |
| 5463 |
return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT); |
0 |
5463 |
return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT); |
0 |
| 5464 |
if (VT == MVT::f128 && C->getValueType(0) == MVT::i128) |
0 |
5464 |
if (VT == MVT::f128 && C->getValueType(0) == MVT::i128) |
0 |
| 5465 |
return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT); |
0 |
5465 |
return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT); |
0 |
| 5466 |
break; |
0 |
5466 |
break; |
0 |
| 5467 |
case ISD::ABS: |
0 |
5467 |
case ISD::ABS: |
0 |
| 5468 |
return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(), |
0 |
5468 |
return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(), |
0 |
| 5469 |
C->isOpaque()); |
0 |
5469 |
C->isOpaque()); |
0 |
| 5470 |
case ISD::BITREVERSE: |
0 |
5470 |
case ISD::BITREVERSE: |
0 |
| 5471 |
return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(), |
0 |
5471 |
return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(), |
0 |
| 5472 |
C->isOpaque()); |
0 |
5472 |
C->isOpaque()); |
0 |
| 5473 |
case ISD::BSWAP: |
0 |
5473 |
case ISD::BSWAP: |
0 |
| 5474 |
return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(), |
0 |
5474 |
return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(), |
0 |
| 5475 |
C->isOpaque()); |
0 |
5475 |
C->isOpaque()); |
0 |
| 5476 |
case ISD::CTPOP: |
0 |
5476 |
case ISD::CTPOP: |
0 |
| 5477 |
return getConstant(Val.popcount(), DL, VT, C->isTargetOpcode(), |
0 |
5477 |
return getConstant(Val.popcount(), DL, VT, C->isTargetOpcode(), |
0 |
| 5478 |
C->isOpaque()); |
0 |
5478 |
C->isOpaque()); |
0 |
| 5479 |
case ISD::CTLZ: |
0 |
5479 |
case ISD::CTLZ: |
0 |
| 5480 |
case ISD::CTLZ_ZERO_UNDEF: |
--- |
5480 |
case ISD::CTLZ_ZERO_UNDEF: |
--- |
| 5481 |
return getConstant(Val.countl_zero(), DL, VT, C->isTargetOpcode(), |
0 |
5481 |
return getConstant(Val.countl_zero(), DL, VT, C->isTargetOpcode(), |
0 |
| 5482 |
C->isOpaque()); |
0 |
5482 |
C->isOpaque()); |
0 |
| 5483 |
case ISD::CTTZ: |
0 |
5483 |
case ISD::CTTZ: |
0 |
| 5484 |
case ISD::CTTZ_ZERO_UNDEF: |
--- |
5484 |
case ISD::CTTZ_ZERO_UNDEF: |
--- |
| 5485 |
return getConstant(Val.countr_zero(), DL, VT, C->isTargetOpcode(), |
0 |
5485 |
return getConstant(Val.countr_zero(), DL, VT, C->isTargetOpcode(), |
0 |
| 5486 |
C->isOpaque()); |
0 |
5486 |
C->isOpaque()); |
0 |
| 5487 |
case ISD::FP16_TO_FP: |
0 |
5487 |
case ISD::FP16_TO_FP: |
0 |
| 5488 |
case ISD::BF16_TO_FP: { |
--- |
5488 |
case ISD::BF16_TO_FP: { |
--- |
| 5489 |
bool Ignored; |
--- |
5489 |
bool Ignored; |
--- |
| 5490 |
APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf() |
0 |
5490 |
APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf() |
0 |
| 5491 |
: APFloat::BFloat(), |
0 |
5491 |
: APFloat::BFloat(), |
0 |
| 5492 |
(Val.getBitWidth() == 16) ? Val : Val.trunc(16)); |
0 |
5492 |
(Val.getBitWidth() == 16) ? Val : Val.trunc(16)); |
0 |
| 5493 |
|
--- |
5493 |
|
--- |
| 5494 |
// This can return overflow, underflow, or inexact; we don't care. |
--- |
5494 |
// This can return overflow, underflow, or inexact; we don't care. |
--- |
| 5495 |
// FIXME need to be more flexible about rounding mode. |
--- |
5495 |
// FIXME need to be more flexible about rounding mode. |
--- |
| 5496 |
(void)FPV.convert(EVTToAPFloatSemantics(VT), |
0 |
5496 |
(void)FPV.convert(EVTToAPFloatSemantics(VT), |
0 |
| 5497 |
APFloat::rmNearestTiesToEven, &Ignored); |
--- |
5497 |
APFloat::rmNearestTiesToEven, &Ignored); |
--- |
| 5498 |
return getConstantFP(FPV, DL, VT); |
0 |
5498 |
return getConstantFP(FPV, DL, VT); |
0 |
| 5499 |
} |
0 |
5499 |
} |
0 |
| 5500 |
case ISD::STEP_VECTOR: { |
0 |
5500 |
case ISD::STEP_VECTOR: { |
0 |
| 5501 |
if (SDValue V = FoldSTEP_VECTOR(DL, VT, N1, *this)) |
0 |
5501 |
if (SDValue V = FoldSTEP_VECTOR(DL, VT, N1, *this)) |
0 |
| 5502 |
return V; |
0 |
5502 |
return V; |
0 |
| 5503 |
break; |
0 |
5503 |
break; |
0 |
| 5504 |
} |
--- |
5504 |
} |
--- |
| 5505 |
} |
--- |
5505 |
} |
--- |
| 5506 |
} |
--- |
5506 |
} |
--- |
| 5507 |
|
--- |
5507 |
|
--- |
| 5508 |
// Constant fold unary operations with a floating point constant operand. |
--- |
5508 |
// Constant fold unary operations with a floating point constant operand. |
--- |
| 5509 |
if (ConstantFPSDNode *C = dyn_cast(N1)) { |
17 |
5509 |
if (ConstantFPSDNode *C = dyn_cast(N1)) { |
17 |
| 5510 |
APFloat V = C->getValueAPF(); // make copy |
0 |
5510 |
APFloat V = C->getValueAPF(); // make copy |
0 |
| 5511 |
switch (Opcode) { |
0 |
5511 |
switch (Opcode) { |
0 |
| 5512 |
case ISD::FNEG: |
0 |
5512 |
case ISD::FNEG: |
0 |
| 5513 |
V.changeSign(); |
0 |
5513 |
V.changeSign(); |
0 |
| 5514 |
return getConstantFP(V, DL, VT); |
0 |
5514 |
return getConstantFP(V, DL, VT); |
0 |
| 5515 |
case ISD::FABS: |
0 |
5515 |
case ISD::FABS: |
0 |
| 5516 |
V.clearSign(); |
0 |
5516 |
V.clearSign(); |
0 |
| 5517 |
return getConstantFP(V, DL, VT); |
0 |
5517 |
return getConstantFP(V, DL, VT); |
0 |
| 5518 |
case ISD::FCEIL: { |
0 |
5518 |
case ISD::FCEIL: { |
0 |
| 5519 |
APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive); |
0 |
5519 |
APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive); |
0 |
| 5520 |
if (fs == APFloat::opOK || fs == APFloat::opInexact) |
0 |
5520 |
if (fs == APFloat::opOK || fs == APFloat::opInexact) |
0 |
| 5521 |
return getConstantFP(V, DL, VT); |
0 |
5521 |
return getConstantFP(V, DL, VT); |
0 |
| 5522 |
break; |
0 |
5522 |
break; |
0 |
| 5523 |
} |
--- |
5523 |
} |
--- |
| 5524 |
case ISD::FTRUNC: { |
0 |
5524 |
case ISD::FTRUNC: { |
0 |
| 5525 |
APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero); |
0 |
5525 |
APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero); |
0 |
| 5526 |
if (fs == APFloat::opOK || fs == APFloat::opInexact) |
0 |
5526 |
if (fs == APFloat::opOK || fs == APFloat::opInexact) |
0 |
| 5527 |
return getConstantFP(V, DL, VT); |
0 |
5527 |
return getConstantFP(V, DL, VT); |
0 |
| 5528 |
break; |
0 |
5528 |
break; |
0 |
| 5529 |
} |
--- |
5529 |
} |
--- |
| 5530 |
case ISD::FFLOOR: { |
0 |
5530 |
case ISD::FFLOOR: { |
0 |
| 5531 |
APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative); |
0 |
5531 |
APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative); |
0 |
| 5532 |
if (fs == APFloat::opOK || fs == APFloat::opInexact) |
0 |
5532 |
if (fs == APFloat::opOK || fs == APFloat::opInexact) |
0 |
| 5533 |
return getConstantFP(V, DL, VT); |
0 |
5533 |
return getConstantFP(V, DL, VT); |
0 |
| 5534 |
break; |
0 |
5534 |
break; |
0 |
| 5535 |
} |
--- |
5535 |
} |
--- |
| 5536 |
case ISD::FP_EXTEND: { |
0 |
5536 |
case ISD::FP_EXTEND: { |
0 |
| 5537 |
bool ignored; |
--- |
5537 |
bool ignored; |
--- |
| 5538 |
// This can return overflow, underflow, or inexact; we don't care. |
--- |
5538 |
// This can return overflow, underflow, or inexact; we don't care. |
--- |
| 5539 |
// FIXME need to be more flexible about rounding mode. |
--- |
5539 |
// FIXME need to be more flexible about rounding mode. |
--- |
| 5540 |
(void)V.convert(EVTToAPFloatSemantics(VT), |
0 |
5540 |
(void)V.convert(EVTToAPFloatSemantics(VT), |
0 |
| 5541 |
APFloat::rmNearestTiesToEven, &ignored); |
--- |
5541 |
APFloat::rmNearestTiesToEven, &ignored); |
--- |
| 5542 |
return getConstantFP(V, DL, VT); |
0 |
5542 |
return getConstantFP(V, DL, VT); |
0 |
| 5543 |
} |
--- |
5543 |
} |
--- |
| 5544 |
case ISD::FP_TO_SINT: |
0 |
5544 |
case ISD::FP_TO_SINT: |
0 |
| 5545 |
case ISD::FP_TO_UINT: { |
--- |
5545 |
case ISD::FP_TO_UINT: { |
--- |
| 5546 |
bool ignored; |
--- |
5546 |
bool ignored; |
--- |
| 5547 |
APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT); |
0 |
5547 |
APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT); |
0 |
| 5548 |
// FIXME need to be more flexible about rounding mode. |
--- |
5548 |
// FIXME need to be more flexible about rounding mode. |
--- |
| 5549 |
APFloat::opStatus s = |
--- |
5549 |
APFloat::opStatus s = |
--- |
| 5550 |
V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored); |
0 |
5550 |
V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored); |
0 |
| 5551 |
if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual |
0 |
5551 |
if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual |
0 |
| 5552 |
break; |
0 |
5552 |
break; |
0 |
| 5553 |
return getConstant(IntVal, DL, VT); |
0 |
5553 |
return getConstant(IntVal, DL, VT); |
0 |
| 5554 |
} |
0 |
5554 |
} |
0 |
| 5555 |
case ISD::BITCAST: |
0 |
5555 |
case ISD::BITCAST: |
0 |
| 5556 |
if (VT == MVT::i16 && C->getValueType(0) == MVT::f16) |
0 |
5556 |
if (VT == MVT::i16 && C->getValueType(0) == MVT::f16) |
0 |
| 5557 |
return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT); |
0 |
5557 |
return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT); |
0 |
| 5558 |
if (VT == MVT::i16 && C->getValueType(0) == MVT::bf16) |
0 |
5558 |
if (VT == MVT::i16 && C->getValueType(0) == MVT::bf16) |
0 |
| 5559 |
return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT); |
0 |
5559 |
return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT); |
0 |
| 5560 |
if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) |
0 |
5560 |
if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) |
0 |
| 5561 |
return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT); |
0 |
5561 |
return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT); |
0 |
| 5562 |
if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) |
0 |
5562 |
if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) |
0 |
| 5563 |
return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT); |
0 |
5563 |
return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT); |
0 |
| 5564 |
break; |
0 |
5564 |
break; |
0 |
| 5565 |
case ISD::FP_TO_FP16: |
0 |
5565 |
case ISD::FP_TO_FP16: |
0 |
| 5566 |
case ISD::FP_TO_BF16: { |
--- |
5566 |
case ISD::FP_TO_BF16: { |
--- |
| 5567 |
bool Ignored; |
--- |
5567 |
bool Ignored; |
--- |
| 5568 |
// This can return overflow, underflow, or inexact; we don't care. |
--- |
5568 |
// This can return overflow, underflow, or inexact; we don't care. |
--- |
| 5569 |
// FIXME need to be more flexible about rounding mode. |
--- |
5569 |
// FIXME need to be more flexible about rounding mode. |
--- |
| 5570 |
(void)V.convert(Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf() |
0 |
5570 |
(void)V.convert(Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf() |
0 |
| 5571 |
: APFloat::BFloat(), |
0 |
5571 |
: APFloat::BFloat(), |
0 |
| 5572 |
APFloat::rmNearestTiesToEven, &Ignored); |
--- |
5572 |
APFloat::rmNearestTiesToEven, &Ignored); |
--- |
| 5573 |
return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT); |
0 |
5573 |
return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT); |
0 |
| 5574 |
} |
--- |
5574 |
} |
--- |
| 5575 |
} |
--- |
5575 |
} |
--- |
| 5576 |
} |
0 |
5576 |
} |
0 |
| 5577 |
|
--- |
5577 |
|
--- |
| 5578 |
// Constant fold unary operations with a vector integer or float operand. |
--- |
5578 |
// Constant fold unary operations with a vector integer or float operand. |
--- |
| 5579 |
switch (Opcode) { |
17 |
5579 |
switch (Opcode) { |
17 |
| 5580 |
default: |
14 |
5580 |
default: |
14 |
| 5581 |
// FIXME: Entirely reasonable to perform folding of other unary |
--- |
5581 |
// FIXME: Entirely reasonable to perform folding of other unary |
--- |
| 5582 |
// operations here as the need arises. |
--- |
5582 |
// operations here as the need arises. |
--- |
| 5583 |
break; |
14 |
5583 |
break; |
14 |
| 5584 |
case ISD::FNEG: |
3 |
5584 |
case ISD::FNEG: |
3 |
| 5585 |
case ISD::FABS: |
--- |
5585 |
case ISD::FABS: |
--- |
| 5586 |
case ISD::FCEIL: |
--- |
5586 |
case ISD::FCEIL: |
--- |
| 5587 |
case ISD::FTRUNC: |
--- |
5587 |
case ISD::FTRUNC: |
--- |
| 5588 |
case ISD::FFLOOR: |
--- |
5588 |
case ISD::FFLOOR: |
--- |
| 5589 |
case ISD::FP_EXTEND: |
--- |
5589 |
case ISD::FP_EXTEND: |
--- |
| 5590 |
case ISD::FP_TO_SINT: |
--- |
5590 |
case ISD::FP_TO_SINT: |
--- |
| 5591 |
case ISD::FP_TO_UINT: |
--- |
5591 |
case ISD::FP_TO_UINT: |
--- |
| 5592 |
case ISD::TRUNCATE: |
--- |
5592 |
case ISD::TRUNCATE: |
--- |
| 5593 |
case ISD::ANY_EXTEND: |
--- |
5593 |
case ISD::ANY_EXTEND: |
--- |
| 5594 |
case ISD::ZERO_EXTEND: |
--- |
5594 |
case ISD::ZERO_EXTEND: |
--- |
| 5595 |
case ISD::SIGN_EXTEND: |
--- |
5595 |
case ISD::SIGN_EXTEND: |
--- |
| 5596 |
case ISD::UINT_TO_FP: |
--- |
5596 |
case ISD::UINT_TO_FP: |
--- |
| 5597 |
case ISD::SINT_TO_FP: |
--- |
5597 |
case ISD::SINT_TO_FP: |
--- |
| 5598 |
case ISD::ABS: |
--- |
5598 |
case ISD::ABS: |
--- |
| 5599 |
case ISD::BITREVERSE: |
--- |
5599 |
case ISD::BITREVERSE: |
--- |
| 5600 |
case ISD::BSWAP: |
--- |
5600 |
case ISD::BSWAP: |
--- |
| 5601 |
case ISD::CTLZ: |
--- |
5601 |
case ISD::CTLZ: |
--- |
| 5602 |
case ISD::CTLZ_ZERO_UNDEF: |
--- |
5602 |
case ISD::CTLZ_ZERO_UNDEF: |
--- |
| 5603 |
case ISD::CTTZ: |
--- |
5603 |
case ISD::CTTZ: |
--- |
| 5604 |
case ISD::CTTZ_ZERO_UNDEF: |
--- |
5604 |
case ISD::CTTZ_ZERO_UNDEF: |
--- |
| 5605 |
case ISD::CTPOP: { |
--- |
5605 |
case ISD::CTPOP: { |
--- |
| 5606 |
SDValue Ops = {N1}; |
3 |
5606 |
SDValue Ops = {N1}; |
3 |
| 5607 |
if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops)) |
3 |
5607 |
if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops)) |
3 |
| 5608 |
return Fold; |
0 |
5608 |
return Fold; |
0 |
| 5609 |
} |
--- |
5609 |
} |
--- |
| 5610 |
} |
--- |
5610 |
} |
--- |
| 5611 |
|
--- |
5611 |
|
--- |
| 5612 |
unsigned OpOpcode = N1.getNode()->getOpcode(); |
17 |
5612 |
unsigned OpOpcode = N1.getNode()->getOpcode(); |
17 |
| 5613 |
switch (Opcode) { |
17 |
5613 |
switch (Opcode) { |
17 |
| 5614 |
case ISD::STEP_VECTOR: |
0 |
5614 |
case ISD::STEP_VECTOR: |
0 |
| 5615 |
assert(VT.isScalableVector() && |
0 |
5615 |
assert(VT.isScalableVector() && |
0 |
| 5616 |
"STEP_VECTOR can only be used with scalable types"); |
--- |
5616 |
"STEP_VECTOR can only be used with scalable types"); |
--- |
| 5617 |
assert(OpOpcode == ISD::TargetConstant && |
0 |
5617 |
assert(OpOpcode == ISD::TargetConstant && |
0 |
| 5618 |
VT.getVectorElementType() == N1.getValueType() && |
--- |
5618 |
VT.getVectorElementType() == N1.getValueType() && |
--- |
| 5619 |
"Unexpected step operand"); |
--- |
5619 |
"Unexpected step operand"); |
--- |
| 5620 |
break; |
0 |
5620 |
break; |
0 |
| 5621 |
case ISD::FREEZE: |
0 |
5621 |
case ISD::FREEZE: |
0 |
| 5622 |
assert(VT == N1.getValueType() && "Unexpected VT!"); |
0 |
5622 |
assert(VT == N1.getValueType() && "Unexpected VT!"); |
0 |
| 5623 |
if (isGuaranteedNotToBeUndefOrPoison(N1, /*PoisonOnly*/ false, |
0 |
5623 |
if (isGuaranteedNotToBeUndefOrPoison(N1, /*PoisonOnly*/ false, |
0 |
| 5624 |
/*Depth*/ 1)) |
--- |
5624 |
/*Depth*/ 1)) |
--- |
| 5625 |
return N1; |
0 |
5625 |
return N1; |
0 |
| 5626 |
break; |
0 |
5626 |
break; |
0 |
| 5627 |
case ISD::TokenFactor: |
14 |
5627 |
case ISD::TokenFactor: |
14 |
| 5628 |
case ISD::MERGE_VALUES: |
--- |
5628 |
case ISD::MERGE_VALUES: |
--- |
| 5629 |
case ISD::CONCAT_VECTORS: |
--- |
5629 |
case ISD::CONCAT_VECTORS: |
--- |
| 5630 |
return N1; // Factor, merge or concat of one node? No need. |
14 |
5630 |
return N1; // Factor, merge or concat of one node? No need. |
14 |
| 5631 |
case ISD::BUILD_VECTOR: { |
0 |
5631 |
case ISD::BUILD_VECTOR: { |
0 |
| 5632 |
// Attempt to simplify BUILD_VECTOR. |
--- |
5632 |
// Attempt to simplify BUILD_VECTOR. |
--- |
| 5633 |
SDValue Ops[] = {N1}; |
0 |
5633 |
SDValue Ops[] = {N1}; |
0 |
| 5634 |
if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this)) |
0 |
5634 |
if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this)) |
0 |
| 5635 |
return V; |
0 |
5635 |
return V; |
0 |
| 5636 |
break; |
0 |
5636 |
break; |
0 |
| 5637 |
} |
--- |
5637 |
} |
--- |
| 5638 |
case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node"); |
0 |
5638 |
case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node"); |
0 |
| 5639 |
case ISD::FP_EXTEND: |
0 |
5639 |
case ISD::FP_EXTEND: |
0 |
| 5640 |
assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() && |
0 |
5640 |
assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() && |
0 |
| 5641 |
"Invalid FP cast!"); |
--- |
5641 |
"Invalid FP cast!"); |
--- |
| 5642 |
if (N1.getValueType() == VT) return N1; // noop conversion. |
0 |
5642 |
if (N1.getValueType() == VT) return N1; // noop conversion. |
0 |
| 5643 |
assert((!VT.isVector() || VT.getVectorElementCount() == |
0 |
5643 |
assert((!VT.isVector() || VT.getVectorElementCount() == |
0 |
| 5644 |
N1.getValueType().getVectorElementCount()) && |
--- |
5644 |
N1.getValueType().getVectorElementCount()) && |
--- |
| 5645 |
"Vector element count mismatch!"); |
--- |
5645 |
"Vector element count mismatch!"); |
--- |
| 5646 |
assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!"); |
0 |
5646 |
assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!"); |
0 |
| 5647 |
if (N1.isUndef()) |
0 |
5647 |
if (N1.isUndef()) |
0 |
| 5648 |
return getUNDEF(VT); |
0 |
5648 |
return getUNDEF(VT); |
0 |
| 5649 |
break; |
0 |
5649 |
break; |
0 |
| 5650 |
case ISD::FP_TO_SINT: |
0 |
5650 |
case ISD::FP_TO_SINT: |
0 |
| 5651 |
case ISD::FP_TO_UINT: |
--- |
5651 |
case ISD::FP_TO_UINT: |
--- |
| 5652 |
if (N1.isUndef()) |
0 |
5652 |
if (N1.isUndef()) |
0 |
| 5653 |
return getUNDEF(VT); |
0 |
5653 |
return getUNDEF(VT); |
0 |
| 5654 |
break; |
0 |
5654 |
break; |
0 |
| 5655 |
case ISD::SINT_TO_FP: |
0 |
5655 |
case ISD::SINT_TO_FP: |
0 |
| 5656 |
case ISD::UINT_TO_FP: |
--- |
5656 |
case ISD::UINT_TO_FP: |
--- |
| 5657 |
// [us]itofp(undef) = 0, because the result value is bounded. |
--- |
5657 |
// [us]itofp(undef) = 0, because the result value is bounded. |
--- |
| 5658 |
if (N1.isUndef()) |
0 |
5658 |
if (N1.isUndef()) |
0 |
| 5659 |
return getConstantFP(0.0, DL, VT); |
0 |
5659 |
return getConstantFP(0.0, DL, VT); |
0 |
| 5660 |
break; |
0 |
5660 |
break; |
0 |
| 5661 |
case ISD::SIGN_EXTEND: |
0 |
5661 |
case ISD::SIGN_EXTEND: |
0 |
| 5662 |
assert(VT.isInteger() && N1.getValueType().isInteger() && |
0 |
5662 |
assert(VT.isInteger() && N1.getValueType().isInteger() && |
0 |
| 5663 |
"Invalid SIGN_EXTEND!"); |
--- |
5663 |
"Invalid SIGN_EXTEND!"); |
--- |
| 5664 |
assert(VT.isVector() == N1.getValueType().isVector() && |
0 |
5664 |
assert(VT.isVector() == N1.getValueType().isVector() && |
0 |
| 5665 |
"SIGN_EXTEND result type type should be vector iff the operand " |
--- |
5665 |
"SIGN_EXTEND result type type should be vector iff the operand " |
--- |
| 5666 |
"type is vector!"); |
--- |
5666 |
"type is vector!"); |
--- |
| 5667 |
if (N1.getValueType() == VT) return N1; // noop extension |
0 |
5667 |
if (N1.getValueType() == VT) return N1; // noop extension |
0 |
| 5668 |
assert((!VT.isVector() || VT.getVectorElementCount() == |
0 |
5668 |
assert((!VT.isVector() || VT.getVectorElementCount() == |
0 |
| 5669 |
N1.getValueType().getVectorElementCount()) && |
--- |
5669 |
N1.getValueType().getVectorElementCount()) && |
--- |
| 5670 |
"Vector element count mismatch!"); |
--- |
5670 |
"Vector element count mismatch!"); |
--- |
| 5671 |
assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!"); |
0 |
5671 |
assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!"); |
0 |
| 5672 |
if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) |
0 |
5672 |
if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) |
0 |
| 5673 |
return getNode(OpOpcode, DL, VT, N1.getOperand(0)); |
0 |
5673 |
return getNode(OpOpcode, DL, VT, N1.getOperand(0)); |
0 |
| 5674 |
if (OpOpcode == ISD::UNDEF) |
0 |
5674 |
if (OpOpcode == ISD::UNDEF) |
0 |
| 5675 |
// sext(undef) = 0, because the top bits will all be the same. |
--- |
5675 |
// sext(undef) = 0, because the top bits will all be the same. |
--- |
| 5676 |
return getConstant(0, DL, VT); |
0 |
5676 |
return getConstant(0, DL, VT); |
0 |
| 5677 |
break; |
0 |
5677 |
break; |
0 |
| 5678 |
case ISD::ZERO_EXTEND: |
1 |
5678 |
case ISD::ZERO_EXTEND: |
1 |
| 5679 |
assert(VT.isInteger() && N1.getValueType().isInteger() && |
1 |
5679 |
assert(VT.isInteger() && N1.getValueType().isInteger() && |
1 |
| 5680 |
"Invalid ZERO_EXTEND!"); |
--- |
5680 |
"Invalid ZERO_EXTEND!"); |
--- |
| 5681 |
assert(VT.isVector() == N1.getValueType().isVector() && |
1 |
5681 |
assert(VT.isVector() == N1.getValueType().isVector() && |
1 |
| 5682 |
"ZERO_EXTEND result type type should be vector iff the operand " |
--- |
5682 |
"ZERO_EXTEND result type type should be vector iff the operand " |
--- |
| 5683 |
"type is vector!"); |
--- |
5683 |
"type is vector!"); |
--- |
| 5684 |
if (N1.getValueType() == VT) return N1; // noop extension |
1 |
5684 |
if (N1.getValueType() == VT) return N1; // noop extension |
1 |
| 5685 |
assert((!VT.isVector() || VT.getVectorElementCount() == |
1 |
5685 |
assert((!VT.isVector() || VT.getVectorElementCount() == |
1 |
| 5686 |
N1.getValueType().getVectorElementCount()) && |
--- |
5686 |
N1.getValueType().getVectorElementCount()) && |
--- |
| 5687 |
"Vector element count mismatch!"); |
--- |
5687 |
"Vector element count mismatch!"); |
--- |
| 5688 |
assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!"); |
1 |
5688 |
assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!"); |
1 |
| 5689 |
if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) |
1 |
5689 |
if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) |
1 |
| 5690 |
return getNode(ISD::ZERO_EXTEND, DL, VT, N1.getOperand(0)); |
0 |
5690 |
return getNode(ISD::ZERO_EXTEND, DL, VT, N1.getOperand(0)); |
0 |
| 5691 |
if (OpOpcode == ISD::UNDEF) |
1 |
5691 |
if (OpOpcode == ISD::UNDEF) |
1 |
| 5692 |
// zext(undef) = 0, because the top bits will be zero. |
--- |
5692 |
// zext(undef) = 0, because the top bits will be zero. |
--- |
| 5693 |
return getConstant(0, DL, VT); |
0 |
5693 |
return getConstant(0, DL, VT); |
0 |
| 5694 |
break; |
1 |
5694 |
break; |
1 |
| 5695 |
case ISD::ANY_EXTEND: |
1 |
5695 |
case ISD::ANY_EXTEND: |
1 |
| 5696 |
assert(VT.isInteger() && N1.getValueType().isInteger() && |
1 |
5696 |
assert(VT.isInteger() && N1.getValueType().isInteger() && |
1 |
| 5697 |
"Invalid ANY_EXTEND!"); |
--- |
5697 |
"Invalid ANY_EXTEND!"); |
--- |
| 5698 |
assert(VT.isVector() == N1.getValueType().isVector() && |
1 |
5698 |
assert(VT.isVector() == N1.getValueType().isVector() && |
1 |
| 5699 |
"ANY_EXTEND result type type should be vector iff the operand " |
--- |
5699 |
"ANY_EXTEND result type type should be vector iff the operand " |
--- |
| 5700 |
"type is vector!"); |
--- |
5700 |
"type is vector!"); |
--- |
| 5701 |
if (N1.getValueType() == VT) return N1; // noop extension |
1 |
5701 |
if (N1.getValueType() == VT) return N1; // noop extension |
1 |
| 5702 |
assert((!VT.isVector() || VT.getVectorElementCount() == |
0 |
5702 |
assert((!VT.isVector() || VT.getVectorElementCount() == |
0 |
| 5703 |
N1.getValueType().getVectorElementCount()) && |
--- |
5703 |
N1.getValueType().getVectorElementCount()) && |
--- |
| 5704 |
"Vector element count mismatch!"); |
--- |
5704 |
"Vector element count mismatch!"); |
--- |
| 5705 |
assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!"); |
0 |
5705 |
assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!"); |
0 |
| 5706 |
|
--- |
5706 |
|
--- |
| 5707 |
if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || |
0 |
5707 |
if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || |
0 |
| 5708 |
OpOpcode == ISD::ANY_EXTEND) |
--- |
5708 |
OpOpcode == ISD::ANY_EXTEND) |
--- |
| 5709 |
// (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) |
--- |
5709 |
// (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) |
--- |
| 5710 |
return getNode(OpOpcode, DL, VT, N1.getOperand(0)); |
0 |
5710 |
return getNode(OpOpcode, DL, VT, N1.getOperand(0)); |
0 |
| 5711 |
if (OpOpcode == ISD::UNDEF) |
0 |
5711 |
if (OpOpcode == ISD::UNDEF) |
0 |
| 5712 |
return getUNDEF(VT); |
0 |
5712 |
return getUNDEF(VT); |
0 |
| 5713 |
|
--- |
5713 |
|
--- |
| 5714 |
// (ext (trunc x)) -> x |
--- |
5714 |
// (ext (trunc x)) -> x |
--- |
| 5715 |
if (OpOpcode == ISD::TRUNCATE) { |
0 |
5715 |
if (OpOpcode == ISD::TRUNCATE) { |
0 |
| 5716 |
SDValue OpOp = N1.getOperand(0); |
0 |
5716 |
SDValue OpOp = N1.getOperand(0); |
0 |
| 5717 |
if (OpOp.getValueType() == VT) { |
0 |
5717 |
if (OpOp.getValueType() == VT) { |
0 |
| 5718 |
transferDbgValues(N1, OpOp); |
0 |
5718 |
transferDbgValues(N1, OpOp); |
0 |
| 5719 |
return OpOp; |
0 |
5719 |
return OpOp; |
0 |
| 5720 |
} |
--- |
5720 |
} |
--- |
| 5721 |
} |
--- |
5721 |
} |
--- |
| 5722 |
break; |
0 |
5722 |
break; |
0 |
| 5723 |
case ISD::TRUNCATE: |
1 |
5723 |
case ISD::TRUNCATE: |
1 |
| 5724 |
assert(VT.isInteger() && N1.getValueType().isInteger() && |
1 |
5724 |
assert(VT.isInteger() && N1.getValueType().isInteger() && |
1 |
| 5725 |
"Invalid TRUNCATE!"); |
--- |
5725 |
"Invalid TRUNCATE!"); |
--- |
| 5726 |
assert(VT.isVector() == N1.getValueType().isVector() && |
1 |
5726 |
assert(VT.isVector() == N1.getValueType().isVector() && |
1 |
| 5727 |
"TRUNCATE result type type should be vector iff the operand " |
--- |
5727 |
"TRUNCATE result type type should be vector iff the operand " |
--- |
| 5728 |
"type is vector!"); |
--- |
5728 |
"type is vector!"); |
--- |
| 5729 |
if (N1.getValueType() == VT) return N1; // noop truncate |
1 |
5729 |
if (N1.getValueType() == VT) return N1; // noop truncate |
1 |
| 5730 |
assert((!VT.isVector() || VT.getVectorElementCount() == |
0 |
5730 |
assert((!VT.isVector() || VT.getVectorElementCount() == |
0 |
| 5731 |
N1.getValueType().getVectorElementCount()) && |
--- |
5731 |
N1.getValueType().getVectorElementCount()) && |
--- |
| 5732 |
"Vector element count mismatch!"); |
--- |
5732 |
"Vector element count mismatch!"); |
--- |
| 5733 |
assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!"); |
0 |
5733 |
assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!"); |
0 |
| 5734 |
if (OpOpcode == ISD::TRUNCATE) |
0 |
5734 |
if (OpOpcode == ISD::TRUNCATE) |
0 |
| 5735 |
return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0)); |
0 |
5735 |
return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0)); |
0 |
| 5736 |
if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || |
0 |
5736 |
if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || |
0 |
| 5737 |
OpOpcode == ISD::ANY_EXTEND) { |
--- |
5737 |
OpOpcode == ISD::ANY_EXTEND) { |
--- |
| 5738 |
// If the source is smaller than the dest, we still need an extend. |
--- |
5738 |
// If the source is smaller than the dest, we still need an extend. |
--- |
| 5739 |
if (N1.getOperand(0).getValueType().getScalarType().bitsLT( |
0 |
5739 |
if (N1.getOperand(0).getValueType().getScalarType().bitsLT( |
0 |
| 5740 |
VT.getScalarType())) |
--- |
5740 |
VT.getScalarType())) |
--- |
| 5741 |
return getNode(OpOpcode, DL, VT, N1.getOperand(0)); |
0 |
5741 |
return getNode(OpOpcode, DL, VT, N1.getOperand(0)); |
0 |
| 5742 |
if (N1.getOperand(0).getValueType().bitsGT(VT)) |
0 |
5742 |
if (N1.getOperand(0).getValueType().bitsGT(VT)) |
0 |
| 5743 |
return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0)); |
0 |
5743 |
return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0)); |
0 |
| 5744 |
return N1.getOperand(0); |
0 |
5744 |
return N1.getOperand(0); |
0 |
| 5745 |
} |
--- |
5745 |
} |
--- |
| 5746 |
if (OpOpcode == ISD::UNDEF) |
0 |
5746 |
if (OpOpcode == ISD::UNDEF) |
0 |
| 5747 |
return getUNDEF(VT); |
0 |
5747 |
return getUNDEF(VT); |
0 |
| 5748 |
if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes) |
0 |
5748 |
if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes) |
0 |
| 5749 |
return getVScale(DL, VT, |
0 |
5749 |
return getVScale(DL, VT, |
0 |
| 5750 |
N1.getConstantOperandAPInt(0).trunc(VT.getSizeInBits())); |
0 |
5750 |
N1.getConstantOperandAPInt(0).trunc(VT.getSizeInBits())); |
0 |
| 5751 |
break; |
0 |
5751 |
break; |
0 |
| 5752 |
case ISD::ANY_EXTEND_VECTOR_INREG: |
0 |
5752 |
case ISD::ANY_EXTEND_VECTOR_INREG: |
0 |
| 5753 |
case ISD::ZERO_EXTEND_VECTOR_INREG: |
--- |
5753 |
case ISD::ZERO_EXTEND_VECTOR_INREG: |
--- |
| 5754 |
case ISD::SIGN_EXTEND_VECTOR_INREG: |
--- |
5754 |
case ISD::SIGN_EXTEND_VECTOR_INREG: |
--- |
| 5755 |
assert(VT.isVector() && "This DAG node is restricted to vector types."); |
0 |
5755 |
assert(VT.isVector() && "This DAG node is restricted to vector types."); |
0 |
| 5756 |
assert(N1.getValueType().bitsLE(VT) && |
0 |
5756 |
assert(N1.getValueType().bitsLE(VT) && |
0 |
| 5757 |
"The input must be the same size or smaller than the result."); |
--- |
5757 |
"The input must be the same size or smaller than the result."); |
--- |
| 5758 |
assert(VT.getVectorMinNumElements() < |
0 |
5758 |
assert(VT.getVectorMinNumElements() < |
0 |
| 5759 |
N1.getValueType().getVectorMinNumElements() && |
--- |
5759 |
N1.getValueType().getVectorMinNumElements() && |
--- |
| 5760 |
"The destination vector type must have fewer lanes than the input."); |
--- |
5760 |
"The destination vector type must have fewer lanes than the input."); |
--- |
| 5761 |
break; |
0 |
5761 |
break; |
0 |
| 5762 |
case ISD::ABS: |
0 |
5762 |
case ISD::ABS: |
0 |
| 5763 |
assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!"); |
0 |
5763 |
assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!"); |
0 |
| 5764 |
if (OpOpcode == ISD::UNDEF) |
0 |
5764 |
if (OpOpcode == ISD::UNDEF) |
0 |
| 5765 |
return getConstant(0, DL, VT); |
0 |
5765 |
return getConstant(0, DL, VT); |
0 |
| 5766 |
break; |
0 |
5766 |
break; |
0 |
| 5767 |
case ISD::BSWAP: |
0 |
5767 |
case ISD::BSWAP: |
0 |
| 5768 |
assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!"); |
0 |
5768 |
assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!"); |
0 |
| 5769 |
assert((VT.getScalarSizeInBits() % 16 == 0) && |
0 |
5769 |
assert((VT.getScalarSizeInBits() % 16 == 0) && |
0 |
| 5770 |
"BSWAP types must be a multiple of 16 bits!"); |
--- |
5770 |
"BSWAP types must be a multiple of 16 bits!"); |
--- |
| 5771 |
if (OpOpcode == ISD::UNDEF) |
0 |
5771 |
if (OpOpcode == ISD::UNDEF) |
0 |
| 5772 |
return getUNDEF(VT); |
0 |
5772 |
return getUNDEF(VT); |
0 |
| 5773 |
// bswap(bswap(X)) -> X. |
--- |
5773 |
// bswap(bswap(X)) -> X. |
--- |
| 5774 |
if (OpOpcode == ISD::BSWAP) |
0 |
5774 |
if (OpOpcode == ISD::BSWAP) |
0 |
| 5775 |
return N1.getOperand(0); |
0 |
5775 |
return N1.getOperand(0); |
0 |
| 5776 |
break; |
0 |
5776 |
break; |
0 |
| 5777 |
case ISD::BITREVERSE: |
0 |
5777 |
case ISD::BITREVERSE: |
0 |
| 5778 |
assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!"); |
0 |
5778 |
assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!"); |
0 |
| 5779 |
if (OpOpcode == ISD::UNDEF) |
0 |
5779 |
if (OpOpcode == ISD::UNDEF) |
0 |
| 5780 |
return getUNDEF(VT); |
0 |
5780 |
return getUNDEF(VT); |
0 |
| 5781 |
break; |
0 |
5781 |
break; |
0 |
| 5782 |
case ISD::BITCAST: |
0 |
5782 |
case ISD::BITCAST: |
0 |
| 5783 |
assert(VT.getSizeInBits() == N1.getValueSizeInBits() && |
0 |
5783 |
assert(VT.getSizeInBits() == N1.getValueSizeInBits() && |
0 |
| 5784 |
"Cannot BITCAST between types of different sizes!"); |
--- |
5784 |
"Cannot BITCAST between types of different sizes!"); |
--- |
| 5785 |
if (VT == N1.getValueType()) return N1; // noop conversion. |
0 |
5785 |
if (VT == N1.getValueType()) return N1; // noop conversion. |
0 |
| 5786 |
if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) |
0 |
5786 |
if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) |
0 |
| 5787 |
return getNode(ISD::BITCAST, DL, VT, N1.getOperand(0)); |
0 |
5787 |
return getNode(ISD::BITCAST, DL, VT, N1.getOperand(0)); |
0 |
| 5788 |
if (OpOpcode == ISD::UNDEF) |
0 |
5788 |
if (OpOpcode == ISD::UNDEF) |
0 |
| 5789 |
return getUNDEF(VT); |
0 |
5789 |
return getUNDEF(VT); |
0 |
| 5790 |
break; |
0 |
5790 |
break; |
0 |
| 5791 |
case ISD::SCALAR_TO_VECTOR: |
0 |
5791 |
case ISD::SCALAR_TO_VECTOR: |
0 |
| 5792 |
assert(VT.isVector() && !N1.getValueType().isVector() && |
0 |
5792 |
assert(VT.isVector() && !N1.getValueType().isVector() && |
0 |
| 5793 |
(VT.getVectorElementType() == N1.getValueType() || |
--- |
5793 |
(VT.getVectorElementType() == N1.getValueType() || |
--- |
| 5794 |
(VT.getVectorElementType().isInteger() && |
--- |
5794 |
(VT.getVectorElementType().isInteger() && |
--- |
| 5795 |
N1.getValueType().isInteger() && |
--- |
5795 |
N1.getValueType().isInteger() && |
--- |
| 5796 |
VT.getVectorElementType().bitsLE(N1.getValueType()))) && |
--- |
5796 |
VT.getVectorElementType().bitsLE(N1.getValueType()))) && |
--- |
| 5797 |
"Illegal SCALAR_TO_VECTOR node!"); |
--- |
5797 |
"Illegal SCALAR_TO_VECTOR node!"); |
--- |
| 5798 |
if (OpOpcode == ISD::UNDEF) |
0 |
5798 |
if (OpOpcode == ISD::UNDEF) |
0 |
| 5799 |
return getUNDEF(VT); |
0 |
5799 |
return getUNDEF(VT); |
0 |
| 5800 |
// scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. |
--- |
5800 |
// scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. |
--- |
| 5801 |
if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && |
0 |
5801 |
if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && |
0 |
| 5802 |
isa(N1.getOperand(1)) && |
0 |
5802 |
isa(N1.getOperand(1)) && |
0 |
| 5803 |
N1.getConstantOperandVal(1) == 0 && |
0 |
5803 |
N1.getConstantOperandVal(1) == 0 && |
0 |
| 5804 |
N1.getOperand(0).getValueType() == VT) |
0 |
5804 |
N1.getOperand(0).getValueType() == VT) |
0 |
| 5805 |
return N1.getOperand(0); |
0 |
5805 |
return N1.getOperand(0); |
0 |
| 5806 |
break; |
0 |
5806 |
break; |
0 |
| 5807 |
case ISD::FNEG: |
0 |
5807 |
case ISD::FNEG: |
0 |
| 5808 |
// Negation of an unknown bag of bits is still completely undefined. |
--- |
5808 |
// Negation of an unknown bag of bits is still completely undefined. |
--- |
| 5809 |
if (OpOpcode == ISD::UNDEF) |
0 |
5809 |
if (OpOpcode == ISD::UNDEF) |
0 |
| 5810 |
return getUNDEF(VT); |
0 |
5810 |
return getUNDEF(VT); |
0 |
| 5811 |
|
--- |
5811 |
|
--- |
| 5812 |
if (OpOpcode == ISD::FNEG) // --X -> X |
0 |
5812 |
if (OpOpcode == ISD::FNEG) // --X -> X |
0 |
| 5813 |
return N1.getOperand(0); |
0 |
5813 |
return N1.getOperand(0); |
0 |
| 5814 |
break; |
0 |
5814 |
break; |
0 |
| 5815 |
case ISD::FABS: |
0 |
5815 |
case ISD::FABS: |
0 |
| 5816 |
if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) |
0 |
5816 |
if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) |
0 |
| 5817 |
return getNode(ISD::FABS, DL, VT, N1.getOperand(0)); |
0 |
5817 |
return getNode(ISD::FABS, DL, VT, N1.getOperand(0)); |
0 |
| 5818 |
break; |
0 |
5818 |
break; |
0 |
| 5819 |
case ISD::VSCALE: |
0 |
5819 |
case ISD::VSCALE: |
0 |
| 5820 |
assert(VT == N1.getValueType() && "Unexpected VT!"); |
0 |
5820 |
assert(VT == N1.getValueType() && "Unexpected VT!"); |
0 |
| 5821 |
break; |
0 |
5821 |
break; |
0 |
| 5822 |
case ISD::CTPOP: |
0 |
5822 |
case ISD::CTPOP: |
0 |
| 5823 |
if (N1.getValueType().getScalarType() == MVT::i1) |
0 |
5823 |
if (N1.getValueType().getScalarType() == MVT::i1) |
0 |
| 5824 |
return N1; |
0 |
5824 |
return N1; |
0 |
| 5825 |
break; |
0 |
5825 |
break; |
0 |
| 5826 |
case ISD::CTLZ: |
0 |
5826 |
case ISD::CTLZ: |
0 |
| 5827 |
case ISD::CTTZ: |
--- |
5827 |
case ISD::CTTZ: |
--- |
| 5828 |
if (N1.getValueType().getScalarType() == MVT::i1) |
0 |
5828 |
if (N1.getValueType().getScalarType() == MVT::i1) |
0 |
| 5829 |
return getNOT(DL, N1, N1.getValueType()); |
0 |
5829 |
return getNOT(DL, N1, N1.getValueType()); |
0 |
| 5830 |
break; |
0 |
5830 |
break; |
0 |
| 5831 |
case ISD::VECREDUCE_ADD: |
0 |
5831 |
case ISD::VECREDUCE_ADD: |
0 |
| 5832 |
if (N1.getValueType().getScalarType() == MVT::i1) |
0 |
5832 |
if (N1.getValueType().getScalarType() == MVT::i1) |
0 |
| 5833 |
return getNode(ISD::VECREDUCE_XOR, DL, VT, N1); |
0 |
5833 |
return getNode(ISD::VECREDUCE_XOR, DL, VT, N1); |
0 |
| 5834 |
break; |
0 |
5834 |
break; |
0 |
| 5835 |
case ISD::VECREDUCE_SMIN: |
0 |
5835 |
case ISD::VECREDUCE_SMIN: |
0 |
| 5836 |
case ISD::VECREDUCE_UMAX: |
--- |
5836 |
case ISD::VECREDUCE_UMAX: |
--- |
| 5837 |
if (N1.getValueType().getScalarType() == MVT::i1) |
0 |
5837 |
if (N1.getValueType().getScalarType() == MVT::i1) |
0 |
| 5838 |
return getNode(ISD::VECREDUCE_OR, DL, VT, N1); |
0 |
5838 |
return getNode(ISD::VECREDUCE_OR, DL, VT, N1); |
0 |
| 5839 |
break; |
0 |
5839 |
break; |
0 |
| 5840 |
case ISD::VECREDUCE_SMAX: |
0 |
5840 |
case ISD::VECREDUCE_SMAX: |
0 |
| 5841 |
case ISD::VECREDUCE_UMIN: |
--- |
5841 |
case ISD::VECREDUCE_UMIN: |
--- |
| 5842 |
if (N1.getValueType().getScalarType() == MVT::i1) |
0 |
5842 |
if (N1.getValueType().getScalarType() == MVT::i1) |
0 |
| 5843 |
return getNode(ISD::VECREDUCE_AND, DL, VT, N1); |
0 |
5843 |
return getNode(ISD::VECREDUCE_AND, DL, VT, N1); |
0 |
| 5844 |
break; |
0 |
5844 |
break; |
0 |
| 5845 |
} |
--- |
5845 |
} |
--- |
| 5846 |
|
--- |
5846 |
|
--- |
| 5847 |
SDNode *N; |
--- |
5847 |
SDNode *N; |
--- |
| 5848 |
SDVTList VTs = getVTList(VT); |
1 |
5848 |
SDVTList VTs = getVTList(VT); |
1 |
| 5849 |
SDValue Ops[] = {N1}; |
1 |
5849 |
SDValue Ops[] = {N1}; |
1 |
| 5850 |
if (VT != MVT::Glue) { // Don't CSE flag producing nodes |
1 |
5850 |
if (VT != MVT::Glue) { // Don't CSE flag producing nodes |
1 |
| 5851 |
FoldingSetNodeID ID; |
1 |
5851 |
FoldingSetNodeID ID; |
1 |
| 5852 |
AddNodeIDNode(ID, Opcode, VTs, Ops); |
1 |
5852 |
AddNodeIDNode(ID, Opcode, VTs, Ops); |
1 |
| 5853 |
void *IP = nullptr; |
1 |
5853 |
void *IP = nullptr; |
1 |
| 5854 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { |
1 |
5854 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { |
1 |
| 5855 |
E->intersectFlagsWith(Flags); |
0 |
5855 |
E->intersectFlagsWith(Flags); |
0 |
| 5856 |
return SDValue(E, 0); |
0 |
5856 |
return SDValue(E, 0); |
0 |
| 5857 |
} |
--- |
5857 |
} |
--- |
| 5858 |
|
--- |
5858 |
|
--- |
| 5859 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
1 |
5859 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
1 |
| 5860 |
N->setFlags(Flags); |
1 |
5860 |
N->setFlags(Flags); |
1 |
| 5861 |
createOperands(N, Ops); |
1 |
5861 |
createOperands(N, Ops); |
1 |
| 5862 |
CSEMap.InsertNode(N, IP); |
1 |
5862 |
CSEMap.InsertNode(N, IP); |
1 |
| 5863 |
} else { |
1 |
5863 |
} else { |
1 |
| 5864 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
0 |
5864 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
0 |
| 5865 |
createOperands(N, Ops); |
0 |
5865 |
createOperands(N, Ops); |
0 |
| 5866 |
} |
--- |
5866 |
} |
--- |
| 5867 |
|
--- |
5867 |
|
--- |
| 5868 |
InsertNode(N); |
1 |
5868 |
InsertNode(N); |
1 |
| 5869 |
SDValue V = SDValue(N, 0); |
1 |
5869 |
SDValue V = SDValue(N, 0); |
1 |
| 5870 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
1 |
5870 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
1 |
| 5871 |
return V; |
1 |
5871 |
return V; |
1 |
| 5872 |
} |
--- |
5872 |
} |
--- |
| 5873 |
|
--- |
5873 |
|
--- |
| 5874 |
static std::optional FoldValue(unsigned Opcode, const APInt &C1, |
0 |
5874 |
static std::optional FoldValue(unsigned Opcode, const APInt &C1, |
0 |
| 5875 |
const APInt &C2) { |
--- |
5875 |
const APInt &C2) { |
--- |
| 5876 |
switch (Opcode) { |
0 |
5876 |
switch (Opcode) { |
0 |
| 5877 |
case ISD::ADD: return C1 + C2; |
0 |
5877 |
case ISD::ADD: return C1 + C2; |
0 |
| 5878 |
case ISD::SUB: return C1 - C2; |
0 |
5878 |
case ISD::SUB: return C1 - C2; |
0 |
| 5879 |
case ISD::MUL: return C1 * C2; |
0 |
5879 |
case ISD::MUL: return C1 * C2; |
0 |
| 5880 |
case ISD::AND: return C1 & C2; |
0 |
5880 |
case ISD::AND: return C1 & C2; |
0 |
| 5881 |
case ISD::OR: return C1 | C2; |
0 |
5881 |
case ISD::OR: return C1 | C2; |
0 |
| 5882 |
case ISD::XOR: return C1 ^ C2; |
0 |
5882 |
case ISD::XOR: return C1 ^ C2; |
0 |
| 5883 |
case ISD::SHL: return C1 << C2; |
0 |
5883 |
case ISD::SHL: return C1 << C2; |
0 |
| 5884 |
case ISD::SRL: return C1.lshr(C2); |
0 |
5884 |
case ISD::SRL: return C1.lshr(C2); |
0 |
| 5885 |
case ISD::SRA: return C1.ashr(C2); |
0 |
5885 |
case ISD::SRA: return C1.ashr(C2); |
0 |
| 5886 |
case ISD::ROTL: return C1.rotl(C2); |
0 |
5886 |
case ISD::ROTL: return C1.rotl(C2); |
0 |
| 5887 |
case ISD::ROTR: return C1.rotr(C2); |
0 |
5887 |
case ISD::ROTR: return C1.rotr(C2); |
0 |
| 5888 |
case ISD::SMIN: return C1.sle(C2) ? C1 : C2; |
0 |
5888 |
case ISD::SMIN: return C1.sle(C2) ? C1 : C2; |
0 |
| 5889 |
case ISD::SMAX: return C1.sge(C2) ? C1 : C2; |
0 |
5889 |
case ISD::SMAX: return C1.sge(C2) ? C1 : C2; |
0 |
| 5890 |
case ISD::UMIN: return C1.ule(C2) ? C1 : C2; |
0 |
5890 |
case ISD::UMIN: return C1.ule(C2) ? C1 : C2; |
0 |
| 5891 |
case ISD::UMAX: return C1.uge(C2) ? C1 : C2; |
0 |
5891 |
case ISD::UMAX: return C1.uge(C2) ? C1 : C2; |
0 |
| 5892 |
case ISD::SADDSAT: return C1.sadd_sat(C2); |
0 |
5892 |
case ISD::SADDSAT: return C1.sadd_sat(C2); |
0 |
| 5893 |
case ISD::UADDSAT: return C1.uadd_sat(C2); |
0 |
5893 |
case ISD::UADDSAT: return C1.uadd_sat(C2); |
0 |
| 5894 |
case ISD::SSUBSAT: return C1.ssub_sat(C2); |
0 |
5894 |
case ISD::SSUBSAT: return C1.ssub_sat(C2); |
0 |
| 5895 |
case ISD::USUBSAT: return C1.usub_sat(C2); |
0 |
5895 |
case ISD::USUBSAT: return C1.usub_sat(C2); |
0 |
| 5896 |
case ISD::SSHLSAT: return C1.sshl_sat(C2); |
0 |
5896 |
case ISD::SSHLSAT: return C1.sshl_sat(C2); |
0 |
| 5897 |
case ISD::USHLSAT: return C1.ushl_sat(C2); |
0 |
5897 |
case ISD::USHLSAT: return C1.ushl_sat(C2); |
0 |
| 5898 |
case ISD::UDIV: |
0 |
5898 |
case ISD::UDIV: |
0 |
| 5899 |
if (!C2.getBoolValue()) |
0 |
5899 |
if (!C2.getBoolValue()) |
0 |
| 5900 |
break; |
0 |
5900 |
break; |
0 |
| 5901 |
return C1.udiv(C2); |
0 |
5901 |
return C1.udiv(C2); |
0 |
| 5902 |
case ISD::UREM: |
0 |
5902 |
case ISD::UREM: |
0 |
| 5903 |
if (!C2.getBoolValue()) |
0 |
5903 |
if (!C2.getBoolValue()) |
0 |
| 5904 |
break; |
0 |
5904 |
break; |
0 |
| 5905 |
return C1.urem(C2); |
0 |
5905 |
return C1.urem(C2); |
0 |
| 5906 |
case ISD::SDIV: |
0 |
5906 |
case ISD::SDIV: |
0 |
| 5907 |
if (!C2.getBoolValue()) |
0 |
5907 |
if (!C2.getBoolValue()) |
0 |
| 5908 |
break; |
0 |
5908 |
break; |
0 |
| 5909 |
return C1.sdiv(C2); |
0 |
5909 |
return C1.sdiv(C2); |
0 |
| 5910 |
case ISD::SREM: |
0 |
5910 |
case ISD::SREM: |
0 |
| 5911 |
if (!C2.getBoolValue()) |
0 |
5911 |
if (!C2.getBoolValue()) |
0 |
| 5912 |
break; |
0 |
5912 |
break; |
0 |
| 5913 |
return C1.srem(C2); |
0 |
5913 |
return C1.srem(C2); |
0 |
| 5914 |
case ISD::MULHS: { |
0 |
5914 |
case ISD::MULHS: { |
0 |
| 5915 |
unsigned FullWidth = C1.getBitWidth() * 2; |
0 |
5915 |
unsigned FullWidth = C1.getBitWidth() * 2; |
0 |
| 5916 |
APInt C1Ext = C1.sext(FullWidth); |
0 |
5916 |
APInt C1Ext = C1.sext(FullWidth); |
0 |
| 5917 |
APInt C2Ext = C2.sext(FullWidth); |
0 |
5917 |
APInt C2Ext = C2.sext(FullWidth); |
0 |
| 5918 |
return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth()); |
0 |
5918 |
return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth()); |
0 |
| 5919 |
} |
0 |
5919 |
} |
0 |
| 5920 |
case ISD::MULHU: { |
0 |
5920 |
case ISD::MULHU: { |
0 |
| 5921 |
unsigned FullWidth = C1.getBitWidth() * 2; |
0 |
5921 |
unsigned FullWidth = C1.getBitWidth() * 2; |
0 |
| 5922 |
APInt C1Ext = C1.zext(FullWidth); |
0 |
5922 |
APInt C1Ext = C1.zext(FullWidth); |
0 |
| 5923 |
APInt C2Ext = C2.zext(FullWidth); |
0 |
5923 |
APInt C2Ext = C2.zext(FullWidth); |
0 |
| 5924 |
return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth()); |
0 |
5924 |
return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth()); |
0 |
| 5925 |
} |
0 |
5925 |
} |
0 |
| 5926 |
case ISD::AVGFLOORS: { |
0 |
5926 |
case ISD::AVGFLOORS: { |
0 |
| 5927 |
unsigned FullWidth = C1.getBitWidth() + 1; |
0 |
5927 |
unsigned FullWidth = C1.getBitWidth() + 1; |
0 |
| 5928 |
APInt C1Ext = C1.sext(FullWidth); |
0 |
5928 |
APInt C1Ext = C1.sext(FullWidth); |
0 |
| 5929 |
APInt C2Ext = C2.sext(FullWidth); |
0 |
5929 |
APInt C2Ext = C2.sext(FullWidth); |
0 |
| 5930 |
return (C1Ext + C2Ext).extractBits(C1.getBitWidth(), 1); |
0 |
5930 |
return (C1Ext + C2Ext).extractBits(C1.getBitWidth(), 1); |
0 |
| 5931 |
} |
0 |
5931 |
} |
0 |
| 5932 |
case ISD::AVGFLOORU: { |
0 |
5932 |
case ISD::AVGFLOORU: { |
0 |
| 5933 |
unsigned FullWidth = C1.getBitWidth() + 1; |
0 |
5933 |
unsigned FullWidth = C1.getBitWidth() + 1; |
0 |
| 5934 |
APInt C1Ext = C1.zext(FullWidth); |
0 |
5934 |
APInt C1Ext = C1.zext(FullWidth); |
0 |
| 5935 |
APInt C2Ext = C2.zext(FullWidth); |
0 |
5935 |
APInt C2Ext = C2.zext(FullWidth); |
0 |
| 5936 |
return (C1Ext + C2Ext).extractBits(C1.getBitWidth(), 1); |
0 |
5936 |
return (C1Ext + C2Ext).extractBits(C1.getBitWidth(), 1); |
0 |
| 5937 |
} |
0 |
5937 |
} |
0 |
| 5938 |
case ISD::AVGCEILS: { |
0 |
5938 |
case ISD::AVGCEILS: { |
0 |
| 5939 |
unsigned FullWidth = C1.getBitWidth() + 1; |
0 |
5939 |
unsigned FullWidth = C1.getBitWidth() + 1; |
0 |
| 5940 |
APInt C1Ext = C1.sext(FullWidth); |
0 |
5940 |
APInt C1Ext = C1.sext(FullWidth); |
0 |
| 5941 |
APInt C2Ext = C2.sext(FullWidth); |
0 |
5941 |
APInt C2Ext = C2.sext(FullWidth); |
0 |
| 5942 |
return (C1Ext + C2Ext + 1).extractBits(C1.getBitWidth(), 1); |
0 |
5942 |
return (C1Ext + C2Ext + 1).extractBits(C1.getBitWidth(), 1); |
0 |
| 5943 |
} |
0 |
5943 |
} |
0 |
| 5944 |
case ISD::AVGCEILU: { |
0 |
5944 |
case ISD::AVGCEILU: { |
0 |
| 5945 |
unsigned FullWidth = C1.getBitWidth() + 1; |
0 |
5945 |
unsigned FullWidth = C1.getBitWidth() + 1; |
0 |
| 5946 |
APInt C1Ext = C1.zext(FullWidth); |
0 |
5946 |
APInt C1Ext = C1.zext(FullWidth); |
0 |
| 5947 |
APInt C2Ext = C2.zext(FullWidth); |
0 |
5947 |
APInt C2Ext = C2.zext(FullWidth); |
0 |
| 5948 |
return (C1Ext + C2Ext + 1).extractBits(C1.getBitWidth(), 1); |
0 |
5948 |
return (C1Ext + C2Ext + 1).extractBits(C1.getBitWidth(), 1); |
0 |
| 5949 |
} |
0 |
5949 |
} |
0 |
| 5950 |
case ISD::ABDS: |
0 |
5950 |
case ISD::ABDS: |
0 |
| 5951 |
return APIntOps::smax(C1, C2) - APIntOps::smin(C1, C2); |
0 |
5951 |
return APIntOps::smax(C1, C2) - APIntOps::smin(C1, C2); |
0 |
| 5952 |
case ISD::ABDU: |
0 |
5952 |
case ISD::ABDU: |
0 |
| 5953 |
return APIntOps::umax(C1, C2) - APIntOps::umin(C1, C2); |
0 |
5953 |
return APIntOps::umax(C1, C2) - APIntOps::umin(C1, C2); |
0 |
| 5954 |
} |
--- |
5954 |
} |
--- |
| 5955 |
return std::nullopt; |
0 |
5955 |
return std::nullopt; |
0 |
| 5956 |
} |
--- |
5956 |
} |
--- |
| 5957 |
|
--- |
5957 |
|
--- |
| 5958 |
// Handle constant folding with UNDEF. |
--- |
5958 |
// Handle constant folding with UNDEF. |
--- |
| 5959 |
// TODO: Handle more cases. |
--- |
5959 |
// TODO: Handle more cases. |
--- |
| 5960 |
static std::optional FoldValueWithUndef(unsigned Opcode, const APInt &C1, |
0 |
5960 |
static std::optional FoldValueWithUndef(unsigned Opcode, const APInt &C1, |
0 |
| 5961 |
bool IsUndef1, const APInt &C2, |
--- |
5961 |
bool IsUndef1, const APInt &C2, |
--- |
| 5962 |
bool IsUndef2) { |
--- |
5962 |
bool IsUndef2) { |
--- |
| 5963 |
if (!(IsUndef1 || IsUndef2)) |
0 |
5963 |
if (!(IsUndef1 || IsUndef2)) |
0 |
| 5964 |
return FoldValue(Opcode, C1, C2); |
0 |
5964 |
return FoldValue(Opcode, C1, C2); |
0 |
| 5965 |
|
--- |
5965 |
|
--- |
| 5966 |
// Fold and(x, undef) -> 0 |
--- |
5966 |
// Fold and(x, undef) -> 0 |
--- |
| 5967 |
// Fold mul(x, undef) -> 0 |
--- |
5967 |
// Fold mul(x, undef) -> 0 |
--- |
| 5968 |
if (Opcode == ISD::AND || Opcode == ISD::MUL) |
0 |
5968 |
if (Opcode == ISD::AND || Opcode == ISD::MUL) |
0 |
| 5969 |
return APInt::getZero(C1.getBitWidth()); |
0 |
5969 |
return APInt::getZero(C1.getBitWidth()); |
0 |
| 5970 |
|
--- |
5970 |
|
--- |
| 5971 |
return std::nullopt; |
0 |
5971 |
return std::nullopt; |
0 |
| 5972 |
} |
--- |
5972 |
} |
--- |
| 5973 |
|
--- |
5973 |
|
--- |
| 5974 |
SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT, |
0 |
5974 |
SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT, |
0 |
| 5975 |
const GlobalAddressSDNode *GA, |
--- |
5975 |
const GlobalAddressSDNode *GA, |
--- |
| 5976 |
const SDNode *N2) { |
--- |
5976 |
const SDNode *N2) { |
--- |
| 5977 |
if (GA->getOpcode() != ISD::GlobalAddress) |
0 |
5977 |
if (GA->getOpcode() != ISD::GlobalAddress) |
0 |
| 5978 |
return SDValue(); |
0 |
5978 |
return SDValue(); |
0 |
| 5979 |
if (!TLI->isOffsetFoldingLegal(GA)) |
0 |
5979 |
if (!TLI->isOffsetFoldingLegal(GA)) |
0 |
| 5980 |
return SDValue(); |
0 |
5980 |
return SDValue(); |
0 |
| 5981 |
auto *C2 = dyn_cast(N2); |
0 |
5981 |
auto *C2 = dyn_cast(N2); |
0 |
| 5982 |
if (!C2) |
0 |
5982 |
if (!C2) |
0 |
| 5983 |
return SDValue(); |
0 |
5983 |
return SDValue(); |
0 |
| 5984 |
int64_t Offset = C2->getSExtValue(); |
0 |
5984 |
int64_t Offset = C2->getSExtValue(); |
0 |
| 5985 |
switch (Opcode) { |
0 |
5985 |
switch (Opcode) { |
0 |
| 5986 |
case ISD::ADD: break; |
0 |
5986 |
case ISD::ADD: break; |
0 |
| 5987 |
case ISD::SUB: Offset = -uint64_t(Offset); break; |
0 |
5987 |
case ISD::SUB: Offset = -uint64_t(Offset); break; |
0 |
| 5988 |
default: return SDValue(); |
0 |
5988 |
default: return SDValue(); |
0 |
| 5989 |
} |
--- |
5989 |
} |
--- |
| 5990 |
return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT, |
0 |
5990 |
return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT, |
0 |
| 5991 |
GA->getOffset() + uint64_t(Offset)); |
0 |
5991 |
GA->getOffset() + uint64_t(Offset)); |
0 |
| 5992 |
} |
--- |
5992 |
} |
--- |
| 5993 |
|
--- |
5993 |
|
--- |
| 5994 |
bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef Ops) { |
14 |
5994 |
bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef Ops) { |
14 |
| 5995 |
switch (Opcode) { |
14 |
5995 |
switch (Opcode) { |
14 |
| 5996 |
case ISD::SDIV: |
0 |
5996 |
case ISD::SDIV: |
0 |
| 5997 |
case ISD::UDIV: |
--- |
5997 |
case ISD::UDIV: |
--- |
| 5998 |
case ISD::SREM: |
--- |
5998 |
case ISD::SREM: |
--- |
| 5999 |
case ISD::UREM: { |
--- |
5999 |
case ISD::UREM: { |
--- |
| 6000 |
// If a divisor is zero/undef or any element of a divisor vector is |
--- |
6000 |
// If a divisor is zero/undef or any element of a divisor vector is |
--- |
| 6001 |
// zero/undef, the whole op is undef. |
--- |
6001 |
// zero/undef, the whole op is undef. |
--- |
| 6002 |
assert(Ops.size() == 2 && "Div/rem should have 2 operands"); |
0 |
6002 |
assert(Ops.size() == 2 && "Div/rem should have 2 operands"); |
0 |
| 6003 |
SDValue Divisor = Ops[1]; |
0 |
6003 |
SDValue Divisor = Ops[1]; |
0 |
| 6004 |
if (Divisor.isUndef() || isNullConstant(Divisor)) |
0 |
6004 |
if (Divisor.isUndef() || isNullConstant(Divisor)) |
0 |
| 6005 |
return true; |
0 |
6005 |
return true; |
0 |
| 6006 |
|
--- |
6006 |
|
--- |
| 6007 |
return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) && |
0 |
6007 |
return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) && |
0 |
| 6008 |
llvm::any_of(Divisor->op_values(), |
0 |
6008 |
llvm::any_of(Divisor->op_values(), |
0 |
| 6009 |
[](SDValue V) { return V.isUndef() || |
0 |
6009 |
[](SDValue V) { return V.isUndef() || |
0 |
| 6010 |
isNullConstant(V); }); |
0 |
6010 |
isNullConstant(V); }); |
0 |
| 6011 |
// TODO: Handle signed overflow. |
--- |
6011 |
// TODO: Handle signed overflow. |
--- |
| 6012 |
} |
--- |
6012 |
} |
--- |
| 6013 |
// TODO: Handle oversized shifts. |
--- |
6013 |
// TODO: Handle oversized shifts. |
--- |
| 6014 |
default: |
14 |
6014 |
default: |
14 |
| 6015 |
return false; |
14 |
6015 |
return false; |
14 |
| 6016 |
} |
--- |
6016 |
} |
--- |
| 6017 |
} |
--- |
6017 |
} |
--- |
| 6018 |
|
--- |
6018 |
|
--- |
| 6019 |
SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, |
14 |
6019 |
SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, |
14 |
| 6020 |
EVT VT, ArrayRef Ops) { |
--- |
6020 |
EVT VT, ArrayRef Ops) { |
--- |
| 6021 |
// If the opcode is a target-specific ISD node, there's nothing we can |
--- |
6021 |
// If the opcode is a target-specific ISD node, there's nothing we can |
--- |
| 6022 |
// do here and the operand rules may not line up with the below, so |
--- |
6022 |
// do here and the operand rules may not line up with the below, so |
--- |
| 6023 |
// bail early. |
--- |
6023 |
// bail early. |
--- |
| 6024 |
// We can't create a scalar CONCAT_VECTORS so skip it. It will break |
--- |
6024 |
// We can't create a scalar CONCAT_VECTORS so skip it. It will break |
--- |
| 6025 |
// for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by |
--- |
6025 |
// for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by |
--- |
| 6026 |
// foldCONCAT_VECTORS in getNode before this is called. |
--- |
6026 |
// foldCONCAT_VECTORS in getNode before this is called. |
--- |
| 6027 |
if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS) |
14 |
6027 |
if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS) |
14 |
| 6028 |
return SDValue(); |
0 |
6028 |
return SDValue(); |
0 |
| 6029 |
|
--- |
6029 |
|
--- |
| 6030 |
unsigned NumOps = Ops.size(); |
14 |
6030 |
unsigned NumOps = Ops.size(); |
14 |
| 6031 |
if (NumOps == 0) |
14 |
6031 |
if (NumOps == 0) |
14 |
| 6032 |
return SDValue(); |
0 |
6032 |
return SDValue(); |
0 |
| 6033 |
|
--- |
6033 |
|
--- |
| 6034 |
if (isUndef(Opcode, Ops)) |
14 |
6034 |
if (isUndef(Opcode, Ops)) |
14 |
| 6035 |
return getUNDEF(VT); |
0 |
6035 |
return getUNDEF(VT); |
0 |
| 6036 |
|
--- |
6036 |
|
--- |
| 6037 |
// Handle binops special cases. |
--- |
6037 |
// Handle binops special cases. |
--- |
| 6038 |
if (NumOps == 2) { |
14 |
6038 |
if (NumOps == 2) { |
14 |
| 6039 |
if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops[0], Ops[1])) |
8 |
6039 |
if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops[0], Ops[1])) |
8 |
| 6040 |
return CFP; |
0 |
6040 |
return CFP; |
0 |
| 6041 |
|
--- |
6041 |
|
--- |
| 6042 |
if (auto *C1 = dyn_cast(Ops[0])) { |
8 |
6042 |
if (auto *C1 = dyn_cast(Ops[0])) { |
8 |
| 6043 |
if (auto *C2 = dyn_cast(Ops[1])) { |
0 |
6043 |
if (auto *C2 = dyn_cast(Ops[1])) { |
0 |
| 6044 |
if (C1->isOpaque() || C2->isOpaque()) |
0 |
6044 |
if (C1->isOpaque() || C2->isOpaque()) |
0 |
| 6045 |
return SDValue(); |
0 |
6045 |
return SDValue(); |
0 |
| 6046 |
|
--- |
6046 |
|
--- |
| 6047 |
std::optional FoldAttempt = |
--- |
6047 |
std::optional FoldAttempt = |
--- |
| 6048 |
FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue()); |
0 |
6048 |
FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue()); |
0 |
| 6049 |
if (!FoldAttempt) |
0 |
6049 |
if (!FoldAttempt) |
0 |
| 6050 |
return SDValue(); |
0 |
6050 |
return SDValue(); |
0 |
| 6051 |
|
--- |
6051 |
|
--- |
| 6052 |
SDValue Folded = getConstant(*FoldAttempt, DL, VT); |
0 |
6052 |
SDValue Folded = getConstant(*FoldAttempt, DL, VT); |
0 |
| 6053 |
assert((!Folded || !VT.isVector()) && |
0 |
6053 |
assert((!Folded || !VT.isVector()) && |
0 |
| 6054 |
"Can't fold vectors ops with scalar operands"); |
--- |
6054 |
"Can't fold vectors ops with scalar operands"); |
--- |
| 6055 |
return Folded; |
0 |
6055 |
return Folded; |
0 |
| 6056 |
} |
0 |
6056 |
} |
0 |
| 6057 |
} |
--- |
6057 |
} |
--- |
| 6058 |
|
--- |
6058 |
|
--- |
| 6059 |
// fold (add Sym, c) -> Sym+c |
--- |
6059 |
// fold (add Sym, c) -> Sym+c |
--- |
| 6060 |
if (GlobalAddressSDNode *GA = dyn_cast(Ops[0])) |
8 |
6060 |
if (GlobalAddressSDNode *GA = dyn_cast(Ops[0])) |
8 |
| 6061 |
return FoldSymbolOffset(Opcode, VT, GA, Ops[1].getNode()); |
0 |
6061 |
return FoldSymbolOffset(Opcode, VT, GA, Ops[1].getNode()); |
0 |
| 6062 |
if (TLI->isCommutativeBinOp(Opcode)) |
8 |
6062 |
if (TLI->isCommutativeBinOp(Opcode)) |
8 |
| 6063 |
if (GlobalAddressSDNode *GA = dyn_cast(Ops[1])) |
5 |
6063 |
if (GlobalAddressSDNode *GA = dyn_cast(Ops[1])) |
5 |
| 6064 |
return FoldSymbolOffset(Opcode, VT, GA, Ops[0].getNode()); |
0 |
6064 |
return FoldSymbolOffset(Opcode, VT, GA, Ops[0].getNode()); |
0 |
| 6065 |
} |
--- |
6065 |
} |
--- |
| 6066 |
|
--- |
6066 |
|
--- |
| 6067 |
// This is for vector folding only from here on. |
--- |
6067 |
// This is for vector folding only from here on. |
--- |
| 6068 |
if (!VT.isVector()) |
14 |
6068 |
if (!VT.isVector()) |
14 |
| 6069 |
return SDValue(); |
14 |
6069 |
return SDValue(); |
14 |
| 6070 |
|
--- |
6070 |
|
--- |
| 6071 |
ElementCount NumElts = VT.getVectorElementCount(); |
0 |
6071 |
ElementCount NumElts = VT.getVectorElementCount(); |
0 |
| 6072 |
|
--- |
6072 |
|
--- |
| 6073 |
// See if we can fold through bitcasted integer ops. |
--- |
6073 |
// See if we can fold through bitcasted integer ops. |
--- |
| 6074 |
if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() && |
0 |
6074 |
if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() && |
0 |
| 6075 |
Ops[0].getValueType() == VT && Ops[1].getValueType() == VT && |
0 |
6075 |
Ops[0].getValueType() == VT && Ops[1].getValueType() == VT && |
0 |
| 6076 |
Ops[0].getOpcode() == ISD::BITCAST && |
0 |
6076 |
Ops[0].getOpcode() == ISD::BITCAST && |
0 |
| 6077 |
Ops[1].getOpcode() == ISD::BITCAST) { |
0 |
6077 |
Ops[1].getOpcode() == ISD::BITCAST) { |
0 |
| 6078 |
SDValue N1 = peekThroughBitcasts(Ops[0]); |
0 |
6078 |
SDValue N1 = peekThroughBitcasts(Ops[0]); |
0 |
| 6079 |
SDValue N2 = peekThroughBitcasts(Ops[1]); |
0 |
6079 |
SDValue N2 = peekThroughBitcasts(Ops[1]); |
0 |
| 6080 |
auto *BV1 = dyn_cast(N1); |
0 |
6080 |
auto *BV1 = dyn_cast(N1); |
0 |
| 6081 |
auto *BV2 = dyn_cast(N2); |
0 |
6081 |
auto *BV2 = dyn_cast(N2); |
0 |
| 6082 |
EVT BVVT = N1.getValueType(); |
0 |
6082 |
EVT BVVT = N1.getValueType(); |
0 |
| 6083 |
if (BV1 && BV2 && BVVT.isInteger() && BVVT == N2.getValueType()) { |
0 |
6083 |
if (BV1 && BV2 && BVVT.isInteger() && BVVT == N2.getValueType()) { |
0 |
| 6084 |
bool IsLE = getDataLayout().isLittleEndian(); |
0 |
6084 |
bool IsLE = getDataLayout().isLittleEndian(); |
0 |
| 6085 |
unsigned EltBits = VT.getScalarSizeInBits(); |
0 |
6085 |
unsigned EltBits = VT.getScalarSizeInBits(); |
0 |
| 6086 |
SmallVector RawBits1, RawBits2; |
0 |
6086 |
SmallVector RawBits1, RawBits2; |
0 |
| 6087 |
BitVector UndefElts1, UndefElts2; |
0 |
6087 |
BitVector UndefElts1, UndefElts2; |
0 |
| 6088 |
if (BV1->getConstantRawBits(IsLE, EltBits, RawBits1, UndefElts1) && |
0 |
6088 |
if (BV1->getConstantRawBits(IsLE, EltBits, RawBits1, UndefElts1) && |
0 |
| 6089 |
BV2->getConstantRawBits(IsLE, EltBits, RawBits2, UndefElts2)) { |
0 |
6089 |
BV2->getConstantRawBits(IsLE, EltBits, RawBits2, UndefElts2)) { |
0 |
| 6090 |
SmallVector RawBits; |
0 |
6090 |
SmallVector RawBits; |
0 |
| 6091 |
for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) { |
0 |
6091 |
for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) { |
0 |
| 6092 |
std::optional Fold = FoldValueWithUndef( |
0 |
6092 |
std::optional Fold = FoldValueWithUndef( |
0 |
| 6093 |
Opcode, RawBits1[I], UndefElts1[I], RawBits2[I], UndefElts2[I]); |
0 |
6093 |
Opcode, RawBits1[I], UndefElts1[I], RawBits2[I], UndefElts2[I]); |
0 |
| 6094 |
if (!Fold) |
0 |
6094 |
if (!Fold) |
0 |
| 6095 |
break; |
0 |
6095 |
break; |
0 |
| 6096 |
RawBits.push_back(*Fold); |
0 |
6096 |
RawBits.push_back(*Fold); |
0 |
| 6097 |
} |
0 |
6097 |
} |
0 |
| 6098 |
if (RawBits.size() == NumElts.getFixedValue()) { |
0 |
6098 |
if (RawBits.size() == NumElts.getFixedValue()) { |
0 |
| 6099 |
// We have constant folded, but we need to cast this again back to |
--- |
6099 |
// We have constant folded, but we need to cast this again back to |
--- |
| 6100 |
// the original (possibly legalized) type. |
--- |
6100 |
// the original (possibly legalized) type. |
--- |
| 6101 |
SmallVector DstBits; |
0 |
6101 |
SmallVector DstBits; |
0 |
| 6102 |
BitVector DstUndefs; |
0 |
6102 |
BitVector DstUndefs; |
0 |
| 6103 |
BuildVectorSDNode::recastRawBits(IsLE, BVVT.getScalarSizeInBits(), |
0 |
6103 |
BuildVectorSDNode::recastRawBits(IsLE, BVVT.getScalarSizeInBits(), |
0 |
| 6104 |
DstBits, RawBits, DstUndefs, |
--- |
6104 |
DstBits, RawBits, DstUndefs, |
--- |
| 6105 |
BitVector(RawBits.size(), false)); |
0 |
6105 |
BitVector(RawBits.size(), false)); |
0 |
| 6106 |
EVT BVEltVT = BV1->getOperand(0).getValueType(); |
0 |
6106 |
EVT BVEltVT = BV1->getOperand(0).getValueType(); |
0 |
| 6107 |
unsigned BVEltBits = BVEltVT.getSizeInBits(); |
0 |
6107 |
unsigned BVEltBits = BVEltVT.getSizeInBits(); |
0 |
| 6108 |
SmallVector Ops(DstBits.size(), getUNDEF(BVEltVT)); |
0 |
6108 |
SmallVector Ops(DstBits.size(), getUNDEF(BVEltVT)); |
0 |
| 6109 |
for (unsigned I = 0, E = DstBits.size(); I != E; ++I) { |
0 |
6109 |
for (unsigned I = 0, E = DstBits.size(); I != E; ++I) { |
0 |
| 6110 |
if (DstUndefs[I]) |
0 |
6110 |
if (DstUndefs[I]) |
0 |
| 6111 |
continue; |
0 |
6111 |
continue; |
0 |
| 6112 |
Ops[I] = getConstant(DstBits[I].sext(BVEltBits), DL, BVEltVT); |
0 |
6112 |
Ops[I] = getConstant(DstBits[I].sext(BVEltBits), DL, BVEltVT); |
0 |
| 6113 |
} |
--- |
6113 |
} |
--- |
| 6114 |
return getBitcast(VT, getBuildVector(BVVT, DL, Ops)); |
0 |
6114 |
return getBitcast(VT, getBuildVector(BVVT, DL, Ops)); |
0 |
| 6115 |
} |
0 |
6115 |
} |
0 |
| 6116 |
} |
0 |
6116 |
} |
0 |
| 6117 |
} |
0 |
6117 |
} |
0 |
| 6118 |
} |
--- |
6118 |
} |
--- |
| 6119 |
|
--- |
6119 |
|
--- |
| 6120 |
// Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)). |
--- |
6120 |
// Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)). |
--- |
| 6121 |
// (shl step_vector(C0), C1) -> (step_vector(C0 << C1)) |
--- |
6121 |
// (shl step_vector(C0), C1) -> (step_vector(C0 << C1)) |
--- |
| 6122 |
if ((Opcode == ISD::MUL || Opcode == ISD::SHL) && |
0 |
6122 |
if ((Opcode == ISD::MUL || Opcode == ISD::SHL) && |
0 |
| 6123 |
Ops[0].getOpcode() == ISD::STEP_VECTOR) { |
0 |
6123 |
Ops[0].getOpcode() == ISD::STEP_VECTOR) { |
0 |
| 6124 |
APInt RHSVal; |
0 |
6124 |
APInt RHSVal; |
0 |
| 6125 |
if (ISD::isConstantSplatVector(Ops[1].getNode(), RHSVal)) { |
0 |
6125 |
if (ISD::isConstantSplatVector(Ops[1].getNode(), RHSVal)) { |
0 |
| 6126 |
APInt NewStep = Opcode == ISD::MUL |
--- |
6126 |
APInt NewStep = Opcode == ISD::MUL |
--- |
| 6127 |
? Ops[0].getConstantOperandAPInt(0) * RHSVal |
0 |
6127 |
? Ops[0].getConstantOperandAPInt(0) * RHSVal |
0 |
| 6128 |
: Ops[0].getConstantOperandAPInt(0) << RHSVal; |
0 |
6128 |
: Ops[0].getConstantOperandAPInt(0) << RHSVal; |
0 |
| 6129 |
return getStepVector(DL, VT, NewStep); |
0 |
6129 |
return getStepVector(DL, VT, NewStep); |
0 |
| 6130 |
} |
0 |
6130 |
} |
0 |
| 6131 |
} |
0 |
6131 |
} |
0 |
| 6132 |
|
--- |
6132 |
|
--- |
| 6133 |
auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) { |
0 |
6133 |
auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) { |
0 |
| 6134 |
return !Op.getValueType().isVector() || |
0 |
6134 |
return !Op.getValueType().isVector() || |
0 |
| 6135 |
Op.getValueType().getVectorElementCount() == NumElts; |
0 |
6135 |
Op.getValueType().getVectorElementCount() == NumElts; |
0 |
| 6136 |
}; |
0 |
6136 |
}; |
0 |
| 6137 |
|
--- |
6137 |
|
--- |
| 6138 |
auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) { |
0 |
6138 |
auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) { |
0 |
| 6139 |
return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE || |
0 |
6139 |
return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE || |
0 |
| 6140 |
Op.getOpcode() == ISD::BUILD_VECTOR || |
0 |
6140 |
Op.getOpcode() == ISD::BUILD_VECTOR || |
0 |
| 6141 |
Op.getOpcode() == ISD::SPLAT_VECTOR; |
0 |
6141 |
Op.getOpcode() == ISD::SPLAT_VECTOR; |
0 |
| 6142 |
}; |
--- |
6142 |
}; |
--- |
| 6143 |
|
--- |
6143 |
|
--- |
| 6144 |
// All operands must be vector types with the same number of elements as |
--- |
6144 |
// All operands must be vector types with the same number of elements as |
--- |
| 6145 |
// the result type and must be either UNDEF or a build/splat vector |
--- |
6145 |
// the result type and must be either UNDEF or a build/splat vector |
--- |
| 6146 |
// or UNDEF scalars. |
--- |
6146 |
// or UNDEF scalars. |
--- |
| 6147 |
if (!llvm::all_of(Ops, IsBuildVectorSplatVectorOrUndef) || |
0 |
6147 |
if (!llvm::all_of(Ops, IsBuildVectorSplatVectorOrUndef) || |
0 |
| 6148 |
!llvm::all_of(Ops, IsScalarOrSameVectorSize)) |
0 |
6148 |
!llvm::all_of(Ops, IsScalarOrSameVectorSize)) |
0 |
| 6149 |
return SDValue(); |
0 |
6149 |
return SDValue(); |
0 |
| 6150 |
|
--- |
6150 |
|
--- |
| 6151 |
// If we are comparing vectors, then the result needs to be a i1 boolean that |
--- |
6151 |
// If we are comparing vectors, then the result needs to be a i1 boolean that |
--- |
| 6152 |
// is then extended back to the legal result type depending on how booleans |
--- |
6152 |
// is then extended back to the legal result type depending on how booleans |
--- |
| 6153 |
// are represented. |
--- |
6153 |
// are represented. |
--- |
| 6154 |
EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType()); |
0 |
6154 |
EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType()); |
0 |
| 6155 |
ISD::NodeType ExtendCode = |
--- |
6155 |
ISD::NodeType ExtendCode = |
--- |
| 6156 |
(Opcode == ISD::SETCC && SVT != VT.getScalarType()) |
0 |
6156 |
(Opcode == ISD::SETCC && SVT != VT.getScalarType()) |
0 |
| 6157 |
? TargetLowering::getExtendForContent(TLI->getBooleanContents(VT)) |
0 |
6157 |
? TargetLowering::getExtendForContent(TLI->getBooleanContents(VT)) |
0 |
| 6158 |
: ISD::SIGN_EXTEND; |
0 |
6158 |
: ISD::SIGN_EXTEND; |
0 |
| 6159 |
|
--- |
6159 |
|
--- |
| 6160 |
// Find legal integer scalar type for constant promotion and |
--- |
6160 |
// Find legal integer scalar type for constant promotion and |
--- |
| 6161 |
// ensure that its scalar size is at least as large as source. |
--- |
6161 |
// ensure that its scalar size is at least as large as source. |
--- |
| 6162 |
EVT LegalSVT = VT.getScalarType(); |
0 |
6162 |
EVT LegalSVT = VT.getScalarType(); |
0 |
| 6163 |
if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) { |
0 |
6163 |
if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) { |
0 |
| 6164 |
LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT); |
0 |
6164 |
LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT); |
0 |
| 6165 |
if (LegalSVT.bitsLT(VT.getScalarType())) |
0 |
6165 |
if (LegalSVT.bitsLT(VT.getScalarType())) |
0 |
| 6166 |
return SDValue(); |
0 |
6166 |
return SDValue(); |
0 |
| 6167 |
} |
--- |
6167 |
} |
--- |
| 6168 |
|
--- |
6168 |
|
--- |
| 6169 |
// For scalable vector types we know we're dealing with SPLAT_VECTORs. We |
--- |
6169 |
// For scalable vector types we know we're dealing with SPLAT_VECTORs. We |
--- |
| 6170 |
// only have one operand to check. For fixed-length vector types we may have |
--- |
6170 |
// only have one operand to check. For fixed-length vector types we may have |
--- |
| 6171 |
// a combination of BUILD_VECTOR and SPLAT_VECTOR. |
--- |
6171 |
// a combination of BUILD_VECTOR and SPLAT_VECTOR. |
--- |
| 6172 |
unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue(); |
0 |
6172 |
unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue(); |
0 |
| 6173 |
|
--- |
6173 |
|
--- |
| 6174 |
// Constant fold each scalar lane separately. |
--- |
6174 |
// Constant fold each scalar lane separately. |
--- |
| 6175 |
SmallVector ScalarResults; |
0 |
6175 |
SmallVector ScalarResults; |
0 |
| 6176 |
for (unsigned I = 0; I != NumVectorElts; I++) { |
0 |
6176 |
for (unsigned I = 0; I != NumVectorElts; I++) { |
0 |
| 6177 |
SmallVector ScalarOps; |
0 |
6177 |
SmallVector ScalarOps; |
0 |
| 6178 |
for (SDValue Op : Ops) { |
0 |
6178 |
for (SDValue Op : Ops) { |
0 |
| 6179 |
EVT InSVT = Op.getValueType().getScalarType(); |
0 |
6179 |
EVT InSVT = Op.getValueType().getScalarType(); |
0 |
| 6180 |
if (Op.getOpcode() != ISD::BUILD_VECTOR && |
0 |
6180 |
if (Op.getOpcode() != ISD::BUILD_VECTOR && |
0 |
| 6181 |
Op.getOpcode() != ISD::SPLAT_VECTOR) { |
0 |
6181 |
Op.getOpcode() != ISD::SPLAT_VECTOR) { |
0 |
| 6182 |
if (Op.isUndef()) |
0 |
6182 |
if (Op.isUndef()) |
0 |
| 6183 |
ScalarOps.push_back(getUNDEF(InSVT)); |
0 |
6183 |
ScalarOps.push_back(getUNDEF(InSVT)); |
0 |
| 6184 |
else |
--- |
6184 |
else |
--- |
| 6185 |
ScalarOps.push_back(Op); |
0 |
6185 |
ScalarOps.push_back(Op); |
0 |
| 6186 |
continue; |
0 |
6186 |
continue; |
0 |
| 6187 |
} |
--- |
6187 |
} |
--- |
| 6188 |
|
--- |
6188 |
|
--- |
| 6189 |
SDValue ScalarOp = |
--- |
6189 |
SDValue ScalarOp = |
--- |
| 6190 |
Op.getOperand(Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I); |
0 |
6190 |
Op.getOperand(Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I); |
0 |
| 6191 |
EVT ScalarVT = ScalarOp.getValueType(); |
0 |
6191 |
EVT ScalarVT = ScalarOp.getValueType(); |
0 |
| 6192 |
|
--- |
6192 |
|
--- |
| 6193 |
// Build vector (integer) scalar operands may need implicit |
--- |
6193 |
// Build vector (integer) scalar operands may need implicit |
--- |
| 6194 |
// truncation - do this before constant folding. |
--- |
6194 |
// truncation - do this before constant folding. |
--- |
| 6195 |
if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT)) { |
0 |
6195 |
if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT)) { |
0 |
| 6196 |
// Don't create illegally-typed nodes unless they're constants or undef |
--- |
6196 |
// Don't create illegally-typed nodes unless they're constants or undef |
--- |
| 6197 |
// - if we fail to constant fold we can't guarantee the (dead) nodes |
--- |
6197 |
// - if we fail to constant fold we can't guarantee the (dead) nodes |
--- |
| 6198 |
// we're creating will be cleaned up before being visited for |
--- |
6198 |
// we're creating will be cleaned up before being visited for |
--- |
| 6199 |
// legalization. |
--- |
6199 |
// legalization. |
--- |
| 6200 |
if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() && |
0 |
6200 |
if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() && |
0 |
| 6201 |
!isa(ScalarOp) && |
0 |
6201 |
!isa(ScalarOp) && |
0 |
| 6202 |
TLI->getTypeAction(*getContext(), InSVT) != |
0 |
6202 |
TLI->getTypeAction(*getContext(), InSVT) != |
0 |
| 6203 |
TargetLowering::TypeLegal) |
--- |
6203 |
TargetLowering::TypeLegal) |
--- |
| 6204 |
return SDValue(); |
0 |
6204 |
return SDValue(); |
0 |
| 6205 |
ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp); |
0 |
6205 |
ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp); |
0 |
| 6206 |
} |
--- |
6206 |
} |
--- |
| 6207 |
|
--- |
6207 |
|
--- |
| 6208 |
ScalarOps.push_back(ScalarOp); |
0 |
6208 |
ScalarOps.push_back(ScalarOp); |
0 |
| 6209 |
} |
--- |
6209 |
} |
--- |
| 6210 |
|
--- |
6210 |
|
--- |
| 6211 |
// Constant fold the scalar operands. |
--- |
6211 |
// Constant fold the scalar operands. |
--- |
| 6212 |
SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps); |
0 |
6212 |
SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps); |
0 |
| 6213 |
|
--- |
6213 |
|
--- |
| 6214 |
// Legalize the (integer) scalar constant if necessary. |
--- |
6214 |
// Legalize the (integer) scalar constant if necessary. |
--- |
| 6215 |
if (LegalSVT != SVT) |
0 |
6215 |
if (LegalSVT != SVT) |
0 |
| 6216 |
ScalarResult = getNode(ExtendCode, DL, LegalSVT, ScalarResult); |
0 |
6216 |
ScalarResult = getNode(ExtendCode, DL, LegalSVT, ScalarResult); |
0 |
| 6217 |
|
--- |
6217 |
|
--- |
| 6218 |
// Scalar folding only succeeded if the result is a constant or UNDEF. |
--- |
6218 |
// Scalar folding only succeeded if the result is a constant or UNDEF. |
--- |
| 6219 |
if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant && |
0 |
6219 |
if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant && |
0 |
| 6220 |
ScalarResult.getOpcode() != ISD::ConstantFP) |
0 |
6220 |
ScalarResult.getOpcode() != ISD::ConstantFP) |
0 |
| 6221 |
return SDValue(); |
0 |
6221 |
return SDValue(); |
0 |
| 6222 |
ScalarResults.push_back(ScalarResult); |
0 |
6222 |
ScalarResults.push_back(ScalarResult); |
0 |
| 6223 |
} |
0 |
6223 |
} |
0 |
| 6224 |
|
--- |
6224 |
|
--- |
| 6225 |
SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, ScalarResults[0]) |
0 |
6225 |
SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, ScalarResults[0]) |
0 |
| 6226 |
: getBuildVector(VT, DL, ScalarResults); |
0 |
6226 |
: getBuildVector(VT, DL, ScalarResults); |
0 |
| 6227 |
NewSDValueDbgMsg(V, "New node fold constant vector: ", this); |
0 |
6227 |
NewSDValueDbgMsg(V, "New node fold constant vector: ", this); |
0 |
| 6228 |
return V; |
0 |
6228 |
return V; |
0 |
| 6229 |
} |
0 |
6229 |
} |
0 |
| 6230 |
|
--- |
6230 |
|
--- |
| 6231 |
SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL, |
8 |
6231 |
SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL, |
8 |
| 6232 |
EVT VT, SDValue N1, SDValue N2) { |
--- |
6232 |
EVT VT, SDValue N1, SDValue N2) { |
--- |
| 6233 |
// TODO: We don't do any constant folding for strict FP opcodes here, but we |
--- |
6233 |
// TODO: We don't do any constant folding for strict FP opcodes here, but we |
--- |
| 6234 |
// should. That will require dealing with a potentially non-default |
--- |
6234 |
// should. That will require dealing with a potentially non-default |
--- |
| 6235 |
// rounding mode, checking the "opStatus" return value from the APFloat |
--- |
6235 |
// rounding mode, checking the "opStatus" return value from the APFloat |
--- |
| 6236 |
// math calculations, and possibly other variations. |
--- |
6236 |
// math calculations, and possibly other variations. |
--- |
| 6237 |
ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, /*AllowUndefs*/ false); |
8 |
6237 |
ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, /*AllowUndefs*/ false); |
8 |
| 6238 |
ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N2, /*AllowUndefs*/ false); |
8 |
6238 |
ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N2, /*AllowUndefs*/ false); |
8 |
| 6239 |
if (N1CFP && N2CFP) { |
8 |
6239 |
if (N1CFP && N2CFP) { |
8 |
| 6240 |
APFloat C1 = N1CFP->getValueAPF(); // make copy |
0 |
6240 |
APFloat C1 = N1CFP->getValueAPF(); // make copy |
0 |
| 6241 |
const APFloat &C2 = N2CFP->getValueAPF(); |
0 |
6241 |
const APFloat &C2 = N2CFP->getValueAPF(); |
0 |
| 6242 |
switch (Opcode) { |
0 |
6242 |
switch (Opcode) { |
0 |
| 6243 |
case ISD::FADD: |
0 |
6243 |
case ISD::FADD: |
0 |
| 6244 |
C1.add(C2, APFloat::rmNearestTiesToEven); |
0 |
6244 |
C1.add(C2, APFloat::rmNearestTiesToEven); |
0 |
| 6245 |
return getConstantFP(C1, DL, VT); |
0 |
6245 |
return getConstantFP(C1, DL, VT); |
0 |
| 6246 |
case ISD::FSUB: |
0 |
6246 |
case ISD::FSUB: |
0 |
| 6247 |
C1.subtract(C2, APFloat::rmNearestTiesToEven); |
0 |
6247 |
C1.subtract(C2, APFloat::rmNearestTiesToEven); |
0 |
| 6248 |
return getConstantFP(C1, DL, VT); |
0 |
6248 |
return getConstantFP(C1, DL, VT); |
0 |
| 6249 |
case ISD::FMUL: |
0 |
6249 |
case ISD::FMUL: |
0 |
| 6250 |
C1.multiply(C2, APFloat::rmNearestTiesToEven); |
0 |
6250 |
C1.multiply(C2, APFloat::rmNearestTiesToEven); |
0 |
| 6251 |
return getConstantFP(C1, DL, VT); |
0 |
6251 |
return getConstantFP(C1, DL, VT); |
0 |
| 6252 |
case ISD::FDIV: |
0 |
6252 |
case ISD::FDIV: |
0 |
| 6253 |
C1.divide(C2, APFloat::rmNearestTiesToEven); |
0 |
6253 |
C1.divide(C2, APFloat::rmNearestTiesToEven); |
0 |
| 6254 |
return getConstantFP(C1, DL, VT); |
0 |
6254 |
return getConstantFP(C1, DL, VT); |
0 |
| 6255 |
case ISD::FREM: |
0 |
6255 |
case ISD::FREM: |
0 |
| 6256 |
C1.mod(C2); |
0 |
6256 |
C1.mod(C2); |
0 |
| 6257 |
return getConstantFP(C1, DL, VT); |
0 |
6257 |
return getConstantFP(C1, DL, VT); |
0 |
| 6258 |
case ISD::FCOPYSIGN: |
0 |
6258 |
case ISD::FCOPYSIGN: |
0 |
| 6259 |
C1.copySign(C2); |
0 |
6259 |
C1.copySign(C2); |
0 |
| 6260 |
return getConstantFP(C1, DL, VT); |
0 |
6260 |
return getConstantFP(C1, DL, VT); |
0 |
| 6261 |
case ISD::FMINNUM: |
0 |
6261 |
case ISD::FMINNUM: |
0 |
| 6262 |
return getConstantFP(minnum(C1, C2), DL, VT); |
0 |
6262 |
return getConstantFP(minnum(C1, C2), DL, VT); |
0 |
| 6263 |
case ISD::FMAXNUM: |
0 |
6263 |
case ISD::FMAXNUM: |
0 |
| 6264 |
return getConstantFP(maxnum(C1, C2), DL, VT); |
0 |
6264 |
return getConstantFP(maxnum(C1, C2), DL, VT); |
0 |
| 6265 |
case ISD::FMINIMUM: |
0 |
6265 |
case ISD::FMINIMUM: |
0 |
| 6266 |
return getConstantFP(minimum(C1, C2), DL, VT); |
0 |
6266 |
return getConstantFP(minimum(C1, C2), DL, VT); |
0 |
| 6267 |
case ISD::FMAXIMUM: |
0 |
6267 |
case ISD::FMAXIMUM: |
0 |
| 6268 |
return getConstantFP(maximum(C1, C2), DL, VT); |
0 |
6268 |
return getConstantFP(maximum(C1, C2), DL, VT); |
0 |
| 6269 |
default: break; |
0 |
6269 |
default: break; |
0 |
| 6270 |
} |
--- |
6270 |
} |
--- |
| 6271 |
} |
0 |
6271 |
} |
0 |
| 6272 |
if (N1CFP && Opcode == ISD::FP_ROUND) { |
8 |
6272 |
if (N1CFP && Opcode == ISD::FP_ROUND) { |
8 |
| 6273 |
APFloat C1 = N1CFP->getValueAPF(); // make copy |
0 |
6273 |
APFloat C1 = N1CFP->getValueAPF(); // make copy |
0 |
| 6274 |
bool Unused; |
--- |
6274 |
bool Unused; |
--- |
| 6275 |
// This can return overflow, underflow, or inexact; we don't care. |
--- |
6275 |
// This can return overflow, underflow, or inexact; we don't care. |
--- |
| 6276 |
// FIXME need to be more flexible about rounding mode. |
--- |
6276 |
// FIXME need to be more flexible about rounding mode. |
--- |
| 6277 |
(void) C1.convert(EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven, |
0 |
6277 |
(void) C1.convert(EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven, |
0 |
| 6278 |
&Unused); |
--- |
6278 |
&Unused); |
--- |
| 6279 |
return getConstantFP(C1, DL, VT); |
0 |
6279 |
return getConstantFP(C1, DL, VT); |
0 |
| 6280 |
} |
0 |
6280 |
} |
0 |
| 6281 |
|
--- |
6281 |
|
--- |
| 6282 |
switch (Opcode) { |
8 |
6282 |
switch (Opcode) { |
8 |
| 6283 |
case ISD::FSUB: |
0 |
6283 |
case ISD::FSUB: |
0 |
| 6284 |
// -0.0 - undef --> undef (consistent with "fneg undef") |
--- |
6284 |
// -0.0 - undef --> undef (consistent with "fneg undef") |
--- |
| 6285 |
if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N1, /*AllowUndefs*/ true)) |
0 |
6285 |
if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N1, /*AllowUndefs*/ true)) |
0 |
| 6286 |
if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef()) |
0 |
6286 |
if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef()) |
0 |
| 6287 |
return getUNDEF(VT); |
0 |
6287 |
return getUNDEF(VT); |
0 |
| 6288 |
[[fallthrough]]; |
--- |
6288 |
[[fallthrough]]; |
--- |
| 6289 |
|
--- |
6289 |
|
--- |
| 6290 |
case ISD::FADD: |
--- |
6290 |
case ISD::FADD: |
--- |
| 6291 |
case ISD::FMUL: |
--- |
6291 |
case ISD::FMUL: |
--- |
| 6292 |
case ISD::FDIV: |
--- |
6292 |
case ISD::FDIV: |
--- |
| 6293 |
case ISD::FREM: |
--- |
6293 |
case ISD::FREM: |
--- |
| 6294 |
// If both operands are undef, the result is undef. If 1 operand is undef, |
--- |
6294 |
// If both operands are undef, the result is undef. If 1 operand is undef, |
--- |
| 6295 |
// the result is NaN. This should match the behavior of the IR optimizer. |
--- |
6295 |
// the result is NaN. This should match the behavior of the IR optimizer. |
--- |
| 6296 |
if (N1.isUndef() && N2.isUndef()) |
0 |
6296 |
if (N1.isUndef() && N2.isUndef()) |
0 |
| 6297 |
return getUNDEF(VT); |
0 |
6297 |
return getUNDEF(VT); |
0 |
| 6298 |
if (N1.isUndef() || N2.isUndef()) |
0 |
6298 |
if (N1.isUndef() || N2.isUndef()) |
0 |
| 6299 |
return getConstantFP(APFloat::getNaN(EVTToAPFloatSemantics(VT)), DL, VT); |
0 |
6299 |
return getConstantFP(APFloat::getNaN(EVTToAPFloatSemantics(VT)), DL, VT); |
0 |
| 6300 |
} |
--- |
6300 |
} |
--- |
| 6301 |
return SDValue(); |
8 |
6301 |
return SDValue(); |
8 |
| 6302 |
} |
--- |
6302 |
} |
--- |
| 6303 |
|
--- |
6303 |
|
--- |
| 6304 |
SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) { |
0 |
6304 |
SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) { |
0 |
| 6305 |
assert(Val.getValueType().isInteger() && "Invalid AssertAlign!"); |
0 |
6305 |
assert(Val.getValueType().isInteger() && "Invalid AssertAlign!"); |
0 |
| 6306 |
|
--- |
6306 |
|
--- |
| 6307 |
// There's no need to assert on a byte-aligned pointer. All pointers are at |
--- |
6307 |
// There's no need to assert on a byte-aligned pointer. All pointers are at |
--- |
| 6308 |
// least byte aligned. |
--- |
6308 |
// least byte aligned. |
--- |
| 6309 |
if (A == Align(1)) |
0 |
6309 |
if (A == Align(1)) |
0 |
| 6310 |
return Val; |
0 |
6310 |
return Val; |
0 |
| 6311 |
|
--- |
6311 |
|
--- |
| 6312 |
FoldingSetNodeID ID; |
0 |
6312 |
FoldingSetNodeID ID; |
0 |
| 6313 |
AddNodeIDNode(ID, ISD::AssertAlign, getVTList(Val.getValueType()), {Val}); |
0 |
6313 |
AddNodeIDNode(ID, ISD::AssertAlign, getVTList(Val.getValueType()), {Val}); |
0 |
| 6314 |
ID.AddInteger(A.value()); |
0 |
6314 |
ID.AddInteger(A.value()); |
0 |
| 6315 |
|
--- |
6315 |
|
--- |
| 6316 |
void *IP = nullptr; |
0 |
6316 |
void *IP = nullptr; |
0 |
| 6317 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) |
0 |
6317 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) |
0 |
| 6318 |
return SDValue(E, 0); |
0 |
6318 |
return SDValue(E, 0); |
0 |
| 6319 |
|
--- |
6319 |
|
--- |
| 6320 |
auto *N = newSDNode(DL.getIROrder(), DL.getDebugLoc(), |
0 |
6320 |
auto *N = newSDNode(DL.getIROrder(), DL.getDebugLoc(), |
0 |
| 6321 |
Val.getValueType(), A); |
0 |
6321 |
Val.getValueType(), A); |
0 |
| 6322 |
createOperands(N, {Val}); |
0 |
6322 |
createOperands(N, {Val}); |
0 |
| 6323 |
|
--- |
6323 |
|
--- |
| 6324 |
CSEMap.InsertNode(N, IP); |
0 |
6324 |
CSEMap.InsertNode(N, IP); |
0 |
| 6325 |
InsertNode(N); |
0 |
6325 |
InsertNode(N); |
0 |
| 6326 |
|
--- |
6326 |
|
--- |
| 6327 |
SDValue V(N, 0); |
0 |
6327 |
SDValue V(N, 0); |
0 |
| 6328 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
6328 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 6329 |
return V; |
0 |
6329 |
return V; |
0 |
| 6330 |
} |
0 |
6330 |
} |
0 |
| 6331 |
|
--- |
6331 |
|
--- |
| 6332 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
5 |
6332 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
5 |
| 6333 |
SDValue N1, SDValue N2) { |
--- |
6333 |
SDValue N1, SDValue N2) { |
--- |
| 6334 |
SDNodeFlags Flags; |
5 |
6334 |
SDNodeFlags Flags; |
5 |
| 6335 |
if (Inserter) |
5 |
6335 |
if (Inserter) |
5 |
| 6336 |
Flags = Inserter->getFlags(); |
0 |
6336 |
Flags = Inserter->getFlags(); |
0 |
| 6337 |
return getNode(Opcode, DL, VT, N1, N2, Flags); |
5 |
6337 |
return getNode(Opcode, DL, VT, N1, N2, Flags); |
5 |
| 6338 |
} |
--- |
6338 |
} |
--- |
| 6339 |
|
--- |
6339 |
|
--- |
| 6340 |
void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1, |
14 |
6340 |
void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1, |
14 |
| 6341 |
SDValue &N2) const { |
--- |
6341 |
SDValue &N2) const { |
--- |
| 6342 |
if (!TLI->isCommutativeBinOp(Opcode)) |
14 |
6342 |
if (!TLI->isCommutativeBinOp(Opcode)) |
14 |
| 6343 |
return; |
3 |
6343 |
return; |
3 |
| 6344 |
|
--- |
6344 |
|
--- |
| 6345 |
// Canonicalize: |
--- |
6345 |
// Canonicalize: |
--- |
| 6346 |
// binop(const, nonconst) -> binop(nonconst, const) |
--- |
6346 |
// binop(const, nonconst) -> binop(nonconst, const) |
--- |
| 6347 |
SDNode *N1C = isConstantIntBuildVectorOrConstantInt(N1); |
11 |
6347 |
SDNode *N1C = isConstantIntBuildVectorOrConstantInt(N1); |
11 |
| 6348 |
SDNode *N2C = isConstantIntBuildVectorOrConstantInt(N2); |
11 |
6348 |
SDNode *N2C = isConstantIntBuildVectorOrConstantInt(N2); |
11 |
| 6349 |
SDNode *N1CFP = isConstantFPBuildVectorOrConstantFP(N1); |
11 |
6349 |
SDNode *N1CFP = isConstantFPBuildVectorOrConstantFP(N1); |
11 |
| 6350 |
SDNode *N2CFP = isConstantFPBuildVectorOrConstantFP(N2); |
11 |
6350 |
SDNode *N2CFP = isConstantFPBuildVectorOrConstantFP(N2); |
11 |
| 6351 |
if ((N1C && !N2C) || (N1CFP && !N2CFP)) |
11 |
6351 |
if ((N1C && !N2C) || (N1CFP && !N2CFP)) |
11 |
| 6352 |
std::swap(N1, N2); |
0 |
6352 |
std::swap(N1, N2); |
0 |
| 6353 |
|
--- |
6353 |
|
--- |
| 6354 |
// Canonicalize: |
--- |
6354 |
// Canonicalize: |
--- |
| 6355 |
// binop(splat(x), step_vector) -> binop(step_vector, splat(x)) |
--- |
6355 |
// binop(splat(x), step_vector) -> binop(step_vector, splat(x)) |
--- |
| 6356 |
else if (N1.getOpcode() == ISD::SPLAT_VECTOR && |
11 |
6356 |
else if (N1.getOpcode() == ISD::SPLAT_VECTOR && |
11 |
| 6357 |
N2.getOpcode() == ISD::STEP_VECTOR) |
0 |
6357 |
N2.getOpcode() == ISD::STEP_VECTOR) |
0 |
| 6358 |
std::swap(N1, N2); |
0 |
6358 |
std::swap(N1, N2); |
0 |
| 6359 |
} |
--- |
6359 |
} |
--- |
| 6360 |
|
--- |
6360 |
|
--- |
| 6361 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
14 |
6361 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
14 |
| 6362 |
SDValue N1, SDValue N2, const SDNodeFlags Flags) { |
--- |
6362 |
SDValue N1, SDValue N2, const SDNodeFlags Flags) { |
--- |
| 6363 |
assert(N1.getOpcode() != ISD::DELETED_NODE && |
14 |
6363 |
assert(N1.getOpcode() != ISD::DELETED_NODE && |
14 |
| 6364 |
N2.getOpcode() != ISD::DELETED_NODE && |
--- |
6364 |
N2.getOpcode() != ISD::DELETED_NODE && |
--- |
| 6365 |
"Operand is DELETED_NODE!"); |
--- |
6365 |
"Operand is DELETED_NODE!"); |
--- |
| 6366 |
|
--- |
6366 |
|
--- |
| 6367 |
canonicalizeCommutativeBinop(Opcode, N1, N2); |
14 |
6367 |
canonicalizeCommutativeBinop(Opcode, N1, N2); |
14 |
| 6368 |
|
--- |
6368 |
|
--- |
| 6369 |
auto *N1C = dyn_cast(N1); |
14 |
6369 |
auto *N1C = dyn_cast(N1); |
14 |
| 6370 |
auto *N2C = dyn_cast(N2); |
14 |
6370 |
auto *N2C = dyn_cast(N2); |
14 |
| 6371 |
|
--- |
6371 |
|
--- |
| 6372 |
// Don't allow undefs in vector splats - we might be returning N2 when folding |
--- |
6372 |
// Don't allow undefs in vector splats - we might be returning N2 when folding |
--- |
| 6373 |
// to zero etc. |
--- |
6373 |
// to zero etc. |
--- |
| 6374 |
ConstantSDNode *N2CV = |
--- |
6374 |
ConstantSDNode *N2CV = |
--- |
| 6375 |
isConstOrConstSplat(N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true); |
14 |
6375 |
isConstOrConstSplat(N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true); |
14 |
| 6376 |
|
--- |
6376 |
|
--- |
| 6377 |
switch (Opcode) { |
14 |
6377 |
switch (Opcode) { |
14 |
| 6378 |
default: break; |
3 |
6378 |
default: break; |
3 |
| 6379 |
case ISD::TokenFactor: |
0 |
6379 |
case ISD::TokenFactor: |
0 |
| 6380 |
assert(VT == MVT::Other && N1.getValueType() == MVT::Other && |
0 |
6380 |
assert(VT == MVT::Other && N1.getValueType() == MVT::Other && |
0 |
| 6381 |
N2.getValueType() == MVT::Other && "Invalid token factor!"); |
--- |
6381 |
N2.getValueType() == MVT::Other && "Invalid token factor!"); |
--- |
| 6382 |
// Fold trivial token factors. |
--- |
6382 |
// Fold trivial token factors. |
--- |
| 6383 |
if (N1.getOpcode() == ISD::EntryToken) return N2; |
0 |
6383 |
if (N1.getOpcode() == ISD::EntryToken) return N2; |
0 |
| 6384 |
if (N2.getOpcode() == ISD::EntryToken) return N1; |
0 |
6384 |
if (N2.getOpcode() == ISD::EntryToken) return N1; |
0 |
| 6385 |
if (N1 == N2) return N1; |
0 |
6385 |
if (N1 == N2) return N1; |
0 |
| 6386 |
break; |
0 |
6386 |
break; |
0 |
| 6387 |
case ISD::BUILD_VECTOR: { |
0 |
6387 |
case ISD::BUILD_VECTOR: { |
0 |
| 6388 |
// Attempt to simplify BUILD_VECTOR. |
--- |
6388 |
// Attempt to simplify BUILD_VECTOR. |
--- |
| 6389 |
SDValue Ops[] = {N1, N2}; |
0 |
6389 |
SDValue Ops[] = {N1, N2}; |
0 |
| 6390 |
if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this)) |
0 |
6390 |
if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this)) |
0 |
| 6391 |
return V; |
0 |
6391 |
return V; |
0 |
| 6392 |
break; |
0 |
6392 |
break; |
0 |
| 6393 |
} |
--- |
6393 |
} |
--- |
| 6394 |
case ISD::CONCAT_VECTORS: { |
0 |
6394 |
case ISD::CONCAT_VECTORS: { |
0 |
| 6395 |
SDValue Ops[] = {N1, N2}; |
0 |
6395 |
SDValue Ops[] = {N1, N2}; |
0 |
| 6396 |
if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this)) |
0 |
6396 |
if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this)) |
0 |
| 6397 |
return V; |
0 |
6397 |
return V; |
0 |
| 6398 |
break; |
0 |
6398 |
break; |
0 |
| 6399 |
} |
--- |
6399 |
} |
--- |
| 6400 |
case ISD::AND: |
1 |
6400 |
case ISD::AND: |
1 |
| 6401 |
assert(VT.isInteger() && "This operator does not apply to FP types!"); |
1 |
6401 |
assert(VT.isInteger() && "This operator does not apply to FP types!"); |
1 |
| 6402 |
assert(N1.getValueType() == N2.getValueType() && |
1 |
6402 |
assert(N1.getValueType() == N2.getValueType() && |
1 |
| 6403 |
N1.getValueType() == VT && "Binary operator types must match!"); |
--- |
6403 |
N1.getValueType() == VT && "Binary operator types must match!"); |
--- |
| 6404 |
// (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's |
--- |
6404 |
// (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's |
--- |
| 6405 |
// worth handling here. |
--- |
6405 |
// worth handling here. |
--- |
| 6406 |
if (N2CV && N2CV->isZero()) |
1 |
6406 |
if (N2CV && N2CV->isZero()) |
1 |
| 6407 |
return N2; |
0 |
6407 |
return N2; |
0 |
| 6408 |
if (N2CV && N2CV->isAllOnes()) // X & -1 -> X |
1 |
6408 |
if (N2CV && N2CV->isAllOnes()) // X & -1 -> X |
1 |
| 6409 |
return N1; |
0 |
6409 |
return N1; |
0 |
| 6410 |
break; |
1 |
6410 |
break; |
1 |
| 6411 |
case ISD::OR: |
10 |
6411 |
case ISD::OR: |
10 |
| 6412 |
case ISD::XOR: |
--- |
6412 |
case ISD::XOR: |
--- |
| 6413 |
case ISD::ADD: |
--- |
6413 |
case ISD::ADD: |
--- |
| 6414 |
case ISD::SUB: |
--- |
6414 |
case ISD::SUB: |
--- |
| 6415 |
assert(VT.isInteger() && "This operator does not apply to FP types!"); |
10 |
6415 |
assert(VT.isInteger() && "This operator does not apply to FP types!"); |
10 |
| 6416 |
assert(N1.getValueType() == N2.getValueType() && |
10 |
6416 |
assert(N1.getValueType() == N2.getValueType() && |
10 |
| 6417 |
N1.getValueType() == VT && "Binary operator types must match!"); |
--- |
6417 |
N1.getValueType() == VT && "Binary operator types must match!"); |
--- |
| 6418 |
// (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so |
--- |
6418 |
// (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so |
--- |
| 6419 |
// it's worth handling here. |
--- |
6419 |
// it's worth handling here. |
--- |
| 6420 |
if (N2CV && N2CV->isZero()) |
10 |
6420 |
if (N2CV && N2CV->isZero()) |
10 |
| 6421 |
return N1; |
9 |
6421 |
return N1; |
9 |
| 6422 |
if ((Opcode == ISD::ADD || Opcode == ISD::SUB) && VT.isVector() && |
1 |
6422 |
if ((Opcode == ISD::ADD || Opcode == ISD::SUB) && VT.isVector() && |
1 |
| 6423 |
VT.getVectorElementType() == MVT::i1) |
1 |
6423 |
VT.getVectorElementType() == MVT::i1) |
1 |
| 6424 |
return getNode(ISD::XOR, DL, VT, N1, N2); |
0 |
6424 |
return getNode(ISD::XOR, DL, VT, N1, N2); |
0 |
| 6425 |
break; |
1 |
6425 |
break; |
1 |
| 6426 |
case ISD::MUL: |
0 |
6426 |
case ISD::MUL: |
0 |
| 6427 |
assert(VT.isInteger() && "This operator does not apply to FP types!"); |
0 |
6427 |
assert(VT.isInteger() && "This operator does not apply to FP types!"); |
0 |
| 6428 |
assert(N1.getValueType() == N2.getValueType() && |
0 |
6428 |
assert(N1.getValueType() == N2.getValueType() && |
0 |
| 6429 |
N1.getValueType() == VT && "Binary operator types must match!"); |
--- |
6429 |
N1.getValueType() == VT && "Binary operator types must match!"); |
--- |
| 6430 |
if (VT.isVector() && VT.getVectorElementType() == MVT::i1) |
0 |
6430 |
if (VT.isVector() && VT.getVectorElementType() == MVT::i1) |
0 |
| 6431 |
return getNode(ISD::AND, DL, VT, N1, N2); |
0 |
6431 |
return getNode(ISD::AND, DL, VT, N1, N2); |
0 |
| 6432 |
if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) { |
0 |
6432 |
if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) { |
0 |
| 6433 |
const APInt &MulImm = N1->getConstantOperandAPInt(0); |
0 |
6433 |
const APInt &MulImm = N1->getConstantOperandAPInt(0); |
0 |
| 6434 |
const APInt &N2CImm = N2C->getAPIntValue(); |
0 |
6434 |
const APInt &N2CImm = N2C->getAPIntValue(); |
0 |
| 6435 |
return getVScale(DL, VT, MulImm * N2CImm); |
0 |
6435 |
return getVScale(DL, VT, MulImm * N2CImm); |
0 |
| 6436 |
} |
--- |
6436 |
} |
--- |
| 6437 |
break; |
0 |
6437 |
break; |
0 |
| 6438 |
case ISD::UDIV: |
0 |
6438 |
case ISD::UDIV: |
0 |
| 6439 |
case ISD::UREM: |
--- |
6439 |
case ISD::UREM: |
--- |
| 6440 |
case ISD::MULHU: |
--- |
6440 |
case ISD::MULHU: |
--- |
| 6441 |
case ISD::MULHS: |
--- |
6441 |
case ISD::MULHS: |
--- |
| 6442 |
case ISD::SDIV: |
--- |
6442 |
case ISD::SDIV: |
--- |
| 6443 |
case ISD::SREM: |
--- |
6443 |
case ISD::SREM: |
--- |
| 6444 |
case ISD::SADDSAT: |
--- |
6444 |
case ISD::SADDSAT: |
--- |
| 6445 |
case ISD::SSUBSAT: |
--- |
6445 |
case ISD::SSUBSAT: |
--- |
| 6446 |
case ISD::UADDSAT: |
--- |
6446 |
case ISD::UADDSAT: |
--- |
| 6447 |
case ISD::USUBSAT: |
--- |
6447 |
case ISD::USUBSAT: |
--- |
| 6448 |
assert(VT.isInteger() && "This operator does not apply to FP types!"); |
0 |
6448 |
assert(VT.isInteger() && "This operator does not apply to FP types!"); |
0 |
| 6449 |
assert(N1.getValueType() == N2.getValueType() && |
0 |
6449 |
assert(N1.getValueType() == N2.getValueType() && |
0 |
| 6450 |
N1.getValueType() == VT && "Binary operator types must match!"); |
--- |
6450 |
N1.getValueType() == VT && "Binary operator types must match!"); |
--- |
| 6451 |
if (VT.isVector() && VT.getVectorElementType() == MVT::i1) { |
0 |
6451 |
if (VT.isVector() && VT.getVectorElementType() == MVT::i1) { |
0 |
| 6452 |
// fold (add_sat x, y) -> (or x, y) for bool types. |
--- |
6452 |
// fold (add_sat x, y) -> (or x, y) for bool types. |
--- |
| 6453 |
if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT) |
0 |
6453 |
if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT) |
0 |
| 6454 |
return getNode(ISD::OR, DL, VT, N1, N2); |
0 |
6454 |
return getNode(ISD::OR, DL, VT, N1, N2); |
0 |
| 6455 |
// fold (sub_sat x, y) -> (and x, ~y) for bool types. |
--- |
6455 |
// fold (sub_sat x, y) -> (and x, ~y) for bool types. |
--- |
| 6456 |
if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT) |
0 |
6456 |
if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT) |
0 |
| 6457 |
return getNode(ISD::AND, DL, VT, N1, getNOT(DL, N2, VT)); |
0 |
6457 |
return getNode(ISD::AND, DL, VT, N1, getNOT(DL, N2, VT)); |
0 |
| 6458 |
} |
--- |
6458 |
} |
--- |
| 6459 |
break; |
0 |
6459 |
break; |
0 |
| 6460 |
case ISD::ABDS: |
0 |
6460 |
case ISD::ABDS: |
0 |
| 6461 |
case ISD::ABDU: |
--- |
6461 |
case ISD::ABDU: |
--- |
| 6462 |
assert(VT.isInteger() && "This operator does not apply to FP types!"); |
0 |
6462 |
assert(VT.isInteger() && "This operator does not apply to FP types!"); |
0 |
| 6463 |
assert(N1.getValueType() == N2.getValueType() && |
0 |
6463 |
assert(N1.getValueType() == N2.getValueType() && |
0 |
| 6464 |
N1.getValueType() == VT && "Binary operator types must match!"); |
--- |
6464 |
N1.getValueType() == VT && "Binary operator types must match!"); |
--- |
| 6465 |
break; |
0 |
6465 |
break; |
0 |
| 6466 |
case ISD::SMIN: |
0 |
6466 |
case ISD::SMIN: |
0 |
| 6467 |
case ISD::UMAX: |
--- |
6467 |
case ISD::UMAX: |
--- |
| 6468 |
assert(VT.isInteger() && "This operator does not apply to FP types!"); |
0 |
6468 |
assert(VT.isInteger() && "This operator does not apply to FP types!"); |
0 |
| 6469 |
assert(N1.getValueType() == N2.getValueType() && |
0 |
6469 |
assert(N1.getValueType() == N2.getValueType() && |
0 |
| 6470 |
N1.getValueType() == VT && "Binary operator types must match!"); |
--- |
6470 |
N1.getValueType() == VT && "Binary operator types must match!"); |
--- |
| 6471 |
if (VT.isVector() && VT.getVectorElementType() == MVT::i1) |
0 |
6471 |
if (VT.isVector() && VT.getVectorElementType() == MVT::i1) |
0 |
| 6472 |
return getNode(ISD::OR, DL, VT, N1, N2); |
0 |
6472 |
return getNode(ISD::OR, DL, VT, N1, N2); |
0 |
| 6473 |
break; |
0 |
6473 |
break; |
0 |
| 6474 |
case ISD::SMAX: |
0 |
6474 |
case ISD::SMAX: |
0 |
| 6475 |
case ISD::UMIN: |
--- |
6475 |
case ISD::UMIN: |
--- |
| 6476 |
assert(VT.isInteger() && "This operator does not apply to FP types!"); |
0 |
6476 |
assert(VT.isInteger() && "This operator does not apply to FP types!"); |
0 |
| 6477 |
assert(N1.getValueType() == N2.getValueType() && |
0 |
6477 |
assert(N1.getValueType() == N2.getValueType() && |
0 |
| 6478 |
N1.getValueType() == VT && "Binary operator types must match!"); |
--- |
6478 |
N1.getValueType() == VT && "Binary operator types must match!"); |
--- |
| 6479 |
if (VT.isVector() && VT.getVectorElementType() == MVT::i1) |
0 |
6479 |
if (VT.isVector() && VT.getVectorElementType() == MVT::i1) |
0 |
| 6480 |
return getNode(ISD::AND, DL, VT, N1, N2); |
0 |
6480 |
return getNode(ISD::AND, DL, VT, N1, N2); |
0 |
| 6481 |
break; |
0 |
6481 |
break; |
0 |
| 6482 |
case ISD::FADD: |
0 |
6482 |
case ISD::FADD: |
0 |
| 6483 |
case ISD::FSUB: |
--- |
6483 |
case ISD::FSUB: |
--- |
| 6484 |
case ISD::FMUL: |
--- |
6484 |
case ISD::FMUL: |
--- |
| 6485 |
case ISD::FDIV: |
--- |
6485 |
case ISD::FDIV: |
--- |
| 6486 |
case ISD::FREM: |
--- |
6486 |
case ISD::FREM: |
--- |
| 6487 |
assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); |
0 |
6487 |
assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); |
0 |
| 6488 |
assert(N1.getValueType() == N2.getValueType() && |
0 |
6488 |
assert(N1.getValueType() == N2.getValueType() && |
0 |
| 6489 |
N1.getValueType() == VT && "Binary operator types must match!"); |
--- |
6489 |
N1.getValueType() == VT && "Binary operator types must match!"); |
--- |
| 6490 |
if (SDValue V = simplifyFPBinop(Opcode, N1, N2, Flags)) |
0 |
6490 |
if (SDValue V = simplifyFPBinop(Opcode, N1, N2, Flags)) |
0 |
| 6491 |
return V; |
0 |
6491 |
return V; |
0 |
| 6492 |
break; |
0 |
6492 |
break; |
0 |
| 6493 |
case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. |
0 |
6493 |
case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. |
0 |
| 6494 |
assert(N1.getValueType() == VT && |
0 |
6494 |
assert(N1.getValueType() == VT && |
0 |
| 6495 |
N1.getValueType().isFloatingPoint() && |
--- |
6495 |
N1.getValueType().isFloatingPoint() && |
--- |
| 6496 |
N2.getValueType().isFloatingPoint() && |
--- |
6496 |
N2.getValueType().isFloatingPoint() && |
--- |
| 6497 |
"Invalid FCOPYSIGN!"); |
--- |
6497 |
"Invalid FCOPYSIGN!"); |
--- |
| 6498 |
break; |
0 |
6498 |
break; |
0 |
| 6499 |
case ISD::SHL: |
0 |
6499 |
case ISD::SHL: |
0 |
| 6500 |
if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) { |
0 |
6500 |
if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) { |
0 |
| 6501 |
const APInt &MulImm = N1->getConstantOperandAPInt(0); |
0 |
6501 |
const APInt &MulImm = N1->getConstantOperandAPInt(0); |
0 |
| 6502 |
const APInt &ShiftImm = N2C->getAPIntValue(); |
0 |
6502 |
const APInt &ShiftImm = N2C->getAPIntValue(); |
0 |
| 6503 |
return getVScale(DL, VT, MulImm << ShiftImm); |
0 |
6503 |
return getVScale(DL, VT, MulImm << ShiftImm); |
0 |
| 6504 |
} |
--- |
6504 |
} |
--- |
| 6505 |
[[fallthrough]]; |
--- |
6505 |
[[fallthrough]]; |
--- |
| 6506 |
case ISD::SRA: |
--- |
6506 |
case ISD::SRA: |
--- |
| 6507 |
case ISD::SRL: |
--- |
6507 |
case ISD::SRL: |
--- |
| 6508 |
if (SDValue V = simplifyShift(N1, N2)) |
0 |
6508 |
if (SDValue V = simplifyShift(N1, N2)) |
0 |
| 6509 |
return V; |
0 |
6509 |
return V; |
0 |
| 6510 |
[[fallthrough]]; |
--- |
6510 |
[[fallthrough]]; |
--- |
| 6511 |
case ISD::ROTL: |
--- |
6511 |
case ISD::ROTL: |
--- |
| 6512 |
case ISD::ROTR: |
--- |
6512 |
case ISD::ROTR: |
--- |
| 6513 |
assert(VT == N1.getValueType() && |
0 |
6513 |
assert(VT == N1.getValueType() && |
0 |
| 6514 |
"Shift operators return type must be the same as their first arg"); |
--- |
6514 |
"Shift operators return type must be the same as their first arg"); |
--- |
| 6515 |
assert(VT.isInteger() && N2.getValueType().isInteger() && |
0 |
6515 |
assert(VT.isInteger() && N2.getValueType().isInteger() && |
0 |
| 6516 |
"Shifts only work on integers"); |
--- |
6516 |
"Shifts only work on integers"); |
--- |
| 6517 |
assert((!VT.isVector() || VT == N2.getValueType()) && |
0 |
6517 |
assert((!VT.isVector() || VT == N2.getValueType()) && |
0 |
| 6518 |
"Vector shift amounts must be in the same as their first arg"); |
--- |
6518 |
"Vector shift amounts must be in the same as their first arg"); |
--- |
| 6519 |
// Verify that the shift amount VT is big enough to hold valid shift |
--- |
6519 |
// Verify that the shift amount VT is big enough to hold valid shift |
--- |
| 6520 |
// amounts. This catches things like trying to shift an i1024 value by an |
--- |
6520 |
// amounts. This catches things like trying to shift an i1024 value by an |
--- |
| 6521 |
// i8, which is easy to fall into in generic code that uses |
--- |
6521 |
// i8, which is easy to fall into in generic code that uses |
--- |
| 6522 |
// TLI.getShiftAmount(). |
--- |
6522 |
// TLI.getShiftAmount(). |
--- |
| 6523 |
assert(N2.getValueType().getScalarSizeInBits() >= |
0 |
6523 |
assert(N2.getValueType().getScalarSizeInBits() >= |
0 |
| 6524 |
Log2_32_Ceil(VT.getScalarSizeInBits()) && |
--- |
6524 |
Log2_32_Ceil(VT.getScalarSizeInBits()) && |
--- |
| 6525 |
"Invalid use of small shift amount with oversized value!"); |
--- |
6525 |
"Invalid use of small shift amount with oversized value!"); |
--- |
| 6526 |
|
--- |
6526 |
|
--- |
| 6527 |
// Always fold shifts of i1 values so the code generator doesn't need to |
--- |
6527 |
// Always fold shifts of i1 values so the code generator doesn't need to |
--- |
| 6528 |
// handle them. Since we know the size of the shift has to be less than the |
--- |
6528 |
// handle them. Since we know the size of the shift has to be less than the |
--- |
| 6529 |
// size of the value, the shift/rotate count is guaranteed to be zero. |
--- |
6529 |
// size of the value, the shift/rotate count is guaranteed to be zero. |
--- |
| 6530 |
if (VT == MVT::i1) |
0 |
6530 |
if (VT == MVT::i1) |
0 |
| 6531 |
return N1; |
0 |
6531 |
return N1; |
0 |
| 6532 |
if (N2CV && N2CV->isZero()) |
0 |
6532 |
if (N2CV && N2CV->isZero()) |
0 |
| 6533 |
return N1; |
0 |
6533 |
return N1; |
0 |
| 6534 |
break; |
0 |
6534 |
break; |
0 |
| 6535 |
case ISD::FP_ROUND: |
0 |
6535 |
case ISD::FP_ROUND: |
0 |
| 6536 |
assert(VT.isFloatingPoint() && |
0 |
6536 |
assert(VT.isFloatingPoint() && |
0 |
| 6537 |
N1.getValueType().isFloatingPoint() && |
--- |
6537 |
N1.getValueType().isFloatingPoint() && |
--- |
| 6538 |
VT.bitsLE(N1.getValueType()) && |
--- |
6538 |
VT.bitsLE(N1.getValueType()) && |
--- |
| 6539 |
N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) && |
--- |
6539 |
N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) && |
--- |
| 6540 |
"Invalid FP_ROUND!"); |
--- |
6540 |
"Invalid FP_ROUND!"); |
--- |
| 6541 |
if (N1.getValueType() == VT) return N1; // noop conversion. |
0 |
6541 |
if (N1.getValueType() == VT) return N1; // noop conversion. |
0 |
| 6542 |
break; |
0 |
6542 |
break; |
0 |
| 6543 |
case ISD::AssertSext: |
0 |
6543 |
case ISD::AssertSext: |
0 |
| 6544 |
case ISD::AssertZext: { |
--- |
6544 |
case ISD::AssertZext: { |
--- |
| 6545 |
EVT EVT = cast(N2)->getVT(); |
0 |
6545 |
EVT EVT = cast(N2)->getVT(); |
0 |
| 6546 |
assert(VT == N1.getValueType() && "Not an inreg extend!"); |
0 |
6546 |
assert(VT == N1.getValueType() && "Not an inreg extend!"); |
0 |
| 6547 |
assert(VT.isInteger() && EVT.isInteger() && |
0 |
6547 |
assert(VT.isInteger() && EVT.isInteger() && |
0 |
| 6548 |
"Cannot *_EXTEND_INREG FP types"); |
--- |
6548 |
"Cannot *_EXTEND_INREG FP types"); |
--- |
| 6549 |
assert(!EVT.isVector() && |
0 |
6549 |
assert(!EVT.isVector() && |
0 |
| 6550 |
"AssertSExt/AssertZExt type should be the vector element type " |
--- |
6550 |
"AssertSExt/AssertZExt type should be the vector element type " |
--- |
| 6551 |
"rather than the vector type!"); |
--- |
6551 |
"rather than the vector type!"); |
--- |
| 6552 |
assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!"); |
0 |
6552 |
assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!"); |
0 |
| 6553 |
if (VT.getScalarType() == EVT) return N1; // noop assertion. |
0 |
6553 |
if (VT.getScalarType() == EVT) return N1; // noop assertion. |
0 |
| 6554 |
break; |
0 |
6554 |
break; |
0 |
| 6555 |
} |
--- |
6555 |
} |
--- |
| 6556 |
case ISD::SIGN_EXTEND_INREG: { |
0 |
6556 |
case ISD::SIGN_EXTEND_INREG: { |
0 |
| 6557 |
EVT EVT = cast(N2)->getVT(); |
0 |
6557 |
EVT EVT = cast(N2)->getVT(); |
0 |
| 6558 |
assert(VT == N1.getValueType() && "Not an inreg extend!"); |
0 |
6558 |
assert(VT == N1.getValueType() && "Not an inreg extend!"); |
0 |
| 6559 |
assert(VT.isInteger() && EVT.isInteger() && |
0 |
6559 |
assert(VT.isInteger() && EVT.isInteger() && |
0 |
| 6560 |
"Cannot *_EXTEND_INREG FP types"); |
--- |
6560 |
"Cannot *_EXTEND_INREG FP types"); |
--- |
| 6561 |
assert(EVT.isVector() == VT.isVector() && |
0 |
6561 |
assert(EVT.isVector() == VT.isVector() && |
0 |
| 6562 |
"SIGN_EXTEND_INREG type should be vector iff the operand " |
--- |
6562 |
"SIGN_EXTEND_INREG type should be vector iff the operand " |
--- |
| 6563 |
"type is vector!"); |
--- |
6563 |
"type is vector!"); |
--- |
| 6564 |
assert((!EVT.isVector() || |
0 |
6564 |
assert((!EVT.isVector() || |
0 |
| 6565 |
EVT.getVectorElementCount() == VT.getVectorElementCount()) && |
--- |
6565 |
EVT.getVectorElementCount() == VT.getVectorElementCount()) && |
--- |
| 6566 |
"Vector element counts must match in SIGN_EXTEND_INREG"); |
--- |
6566 |
"Vector element counts must match in SIGN_EXTEND_INREG"); |
--- |
| 6567 |
assert(EVT.bitsLE(VT) && "Not extending!"); |
0 |
6567 |
assert(EVT.bitsLE(VT) && "Not extending!"); |
0 |
| 6568 |
if (EVT == VT) return N1; // Not actually extending |
0 |
6568 |
if (EVT == VT) return N1; // Not actually extending |
0 |
| 6569 |
|
--- |
6569 |
|
--- |
| 6570 |
auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) { |
0 |
6570 |
auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) { |
0 |
| 6571 |
unsigned FromBits = EVT.getScalarSizeInBits(); |
0 |
6571 |
unsigned FromBits = EVT.getScalarSizeInBits(); |
0 |
| 6572 |
Val <<= Val.getBitWidth() - FromBits; |
0 |
6572 |
Val <<= Val.getBitWidth() - FromBits; |
0 |
| 6573 |
Val.ashrInPlace(Val.getBitWidth() - FromBits); |
0 |
6573 |
Val.ashrInPlace(Val.getBitWidth() - FromBits); |
0 |
| 6574 |
return getConstant(Val, DL, ConstantVT); |
0 |
6574 |
return getConstant(Val, DL, ConstantVT); |
0 |
| 6575 |
}; |
0 |
6575 |
}; |
0 |
| 6576 |
|
--- |
6576 |
|
--- |
| 6577 |
if (N1C) { |
0 |
6577 |
if (N1C) { |
0 |
| 6578 |
const APInt &Val = N1C->getAPIntValue(); |
0 |
6578 |
const APInt &Val = N1C->getAPIntValue(); |
0 |
| 6579 |
return SignExtendInReg(Val, VT); |
0 |
6579 |
return SignExtendInReg(Val, VT); |
0 |
| 6580 |
} |
--- |
6580 |
} |
--- |
| 6581 |
|
--- |
6581 |
|
--- |
| 6582 |
if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) { |
0 |
6582 |
if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) { |
0 |
| 6583 |
SmallVector Ops; |
0 |
6583 |
SmallVector Ops; |
0 |
| 6584 |
llvm::EVT OpVT = N1.getOperand(0).getValueType(); |
0 |
6584 |
llvm::EVT OpVT = N1.getOperand(0).getValueType(); |
0 |
| 6585 |
for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) { |
0 |
6585 |
for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) { |
0 |
| 6586 |
SDValue Op = N1.getOperand(i); |
0 |
6586 |
SDValue Op = N1.getOperand(i); |
0 |
| 6587 |
if (Op.isUndef()) { |
0 |
6587 |
if (Op.isUndef()) { |
0 |
| 6588 |
Ops.push_back(getUNDEF(OpVT)); |
0 |
6588 |
Ops.push_back(getUNDEF(OpVT)); |
0 |
| 6589 |
continue; |
0 |
6589 |
continue; |
0 |
| 6590 |
} |
--- |
6590 |
} |
--- |
| 6591 |
ConstantSDNode *C = cast(Op); |
0 |
6591 |
ConstantSDNode *C = cast(Op); |
0 |
| 6592 |
APInt Val = C->getAPIntValue(); |
0 |
6592 |
APInt Val = C->getAPIntValue(); |
0 |
| 6593 |
Ops.push_back(SignExtendInReg(Val, OpVT)); |
0 |
6593 |
Ops.push_back(SignExtendInReg(Val, OpVT)); |
0 |
| 6594 |
} |
0 |
6594 |
} |
0 |
| 6595 |
return getBuildVector(VT, DL, Ops); |
0 |
6595 |
return getBuildVector(VT, DL, Ops); |
0 |
| 6596 |
} |
0 |
6596 |
} |
0 |
| 6597 |
break; |
0 |
6597 |
break; |
0 |
| 6598 |
} |
--- |
6598 |
} |
--- |
| 6599 |
case ISD::FP_TO_SINT_SAT: |
0 |
6599 |
case ISD::FP_TO_SINT_SAT: |
0 |
| 6600 |
case ISD::FP_TO_UINT_SAT: { |
--- |
6600 |
case ISD::FP_TO_UINT_SAT: { |
--- |
| 6601 |
assert(VT.isInteger() && cast(N2)->getVT().isInteger() && |
0 |
6601 |
assert(VT.isInteger() && cast(N2)->getVT().isInteger() && |
0 |
| 6602 |
N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT"); |
--- |
6602 |
N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT"); |
--- |
| 6603 |
assert(N1.getValueType().isVector() == VT.isVector() && |
0 |
6603 |
assert(N1.getValueType().isVector() == VT.isVector() && |
0 |
| 6604 |
"FP_TO_*INT_SAT type should be vector iff the operand type is " |
--- |
6604 |
"FP_TO_*INT_SAT type should be vector iff the operand type is " |
--- |
| 6605 |
"vector!"); |
--- |
6605 |
"vector!"); |
--- |
| 6606 |
assert((!VT.isVector() || VT.getVectorElementCount() == |
0 |
6606 |
assert((!VT.isVector() || VT.getVectorElementCount() == |
0 |
| 6607 |
N1.getValueType().getVectorElementCount()) && |
--- |
6607 |
N1.getValueType().getVectorElementCount()) && |
--- |
| 6608 |
"Vector element counts must match in FP_TO_*INT_SAT"); |
--- |
6608 |
"Vector element counts must match in FP_TO_*INT_SAT"); |
--- |
| 6609 |
assert(!cast(N2)->getVT().isVector() && |
0 |
6609 |
assert(!cast(N2)->getVT().isVector() && |
0 |
| 6610 |
"Type to saturate to must be a scalar."); |
--- |
6610 |
"Type to saturate to must be a scalar."); |
--- |
| 6611 |
assert(cast(N2)->getVT().bitsLE(VT.getScalarType()) && |
0 |
6611 |
assert(cast(N2)->getVT().bitsLE(VT.getScalarType()) && |
0 |
| 6612 |
"Not extending!"); |
--- |
6612 |
"Not extending!"); |
--- |
| 6613 |
break; |
0 |
6613 |
break; |
0 |
| 6614 |
} |
--- |
6614 |
} |
--- |
| 6615 |
case ISD::EXTRACT_VECTOR_ELT: |
0 |
6615 |
case ISD::EXTRACT_VECTOR_ELT: |
0 |
| 6616 |
assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() && |
0 |
6616 |
assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() && |
0 |
| 6617 |
"The result of EXTRACT_VECTOR_ELT must be at least as wide as the \ |
--- |
6617 |
"The result of EXTRACT_VECTOR_ELT must be at least as wide as the \ |
--- |
| 6618 |
element type of the vector."); |
--- |
6618 |
element type of the vector."); |
--- |
| 6619 |
|
--- |
6619 |
|
--- |
| 6620 |
// Extract from an undefined value or using an undefined index is undefined. |
--- |
6620 |
// Extract from an undefined value or using an undefined index is undefined. |
--- |
| 6621 |
if (N1.isUndef() || N2.isUndef()) |
0 |
6621 |
if (N1.isUndef() || N2.isUndef()) |
0 |
| 6622 |
return getUNDEF(VT); |
0 |
6622 |
return getUNDEF(VT); |
0 |
| 6623 |
|
--- |
6623 |
|
--- |
| 6624 |
// EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length |
--- |
6624 |
// EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length |
--- |
| 6625 |
// vectors. For scalable vectors we will provide appropriate support for |
--- |
6625 |
// vectors. For scalable vectors we will provide appropriate support for |
--- |
| 6626 |
// dealing with arbitrary indices. |
--- |
6626 |
// dealing with arbitrary indices. |
--- |
| 6627 |
if (N2C && N1.getValueType().isFixedLengthVector() && |
0 |
6627 |
if (N2C && N1.getValueType().isFixedLengthVector() && |
0 |
| 6628 |
N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements())) |
0 |
6628 |
N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements())) |
0 |
| 6629 |
return getUNDEF(VT); |
0 |
6629 |
return getUNDEF(VT); |
0 |
| 6630 |
|
--- |
6630 |
|
--- |
| 6631 |
// EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is |
--- |
6631 |
// EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is |
--- |
| 6632 |
// expanding copies of large vectors from registers. This only works for |
--- |
6632 |
// expanding copies of large vectors from registers. This only works for |
--- |
| 6633 |
// fixed length vectors, since we need to know the exact number of |
--- |
6633 |
// fixed length vectors, since we need to know the exact number of |
--- |
| 6634 |
// elements. |
--- |
6634 |
// elements. |
--- |
| 6635 |
if (N2C && N1.getOperand(0).getValueType().isFixedLengthVector() && |
0 |
6635 |
if (N2C && N1.getOperand(0).getValueType().isFixedLengthVector() && |
0 |
| 6636 |
N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0) { |
0 |
6636 |
N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0) { |
0 |
| 6637 |
unsigned Factor = |
--- |
6637 |
unsigned Factor = |
--- |
| 6638 |
N1.getOperand(0).getValueType().getVectorNumElements(); |
0 |
6638 |
N1.getOperand(0).getValueType().getVectorNumElements(); |
0 |
| 6639 |
return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, |
0 |
6639 |
return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, |
0 |
| 6640 |
N1.getOperand(N2C->getZExtValue() / Factor), |
0 |
6640 |
N1.getOperand(N2C->getZExtValue() / Factor), |
0 |
| 6641 |
getVectorIdxConstant(N2C->getZExtValue() % Factor, DL)); |
0 |
6641 |
getVectorIdxConstant(N2C->getZExtValue() % Factor, DL)); |
0 |
| 6642 |
} |
--- |
6642 |
} |
--- |
| 6643 |
|
--- |
6643 |
|
--- |
| 6644 |
// EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while |
--- |
6644 |
// EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while |
--- |
| 6645 |
// lowering is expanding large vector constants. |
--- |
6645 |
// lowering is expanding large vector constants. |
--- |
| 6646 |
if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR || |
0 |
6646 |
if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR || |
0 |
| 6647 |
N1.getOpcode() == ISD::SPLAT_VECTOR)) { |
0 |
6647 |
N1.getOpcode() == ISD::SPLAT_VECTOR)) { |
0 |
| 6648 |
assert((N1.getOpcode() != ISD::BUILD_VECTOR || |
0 |
6648 |
assert((N1.getOpcode() != ISD::BUILD_VECTOR || |
0 |
| 6649 |
N1.getValueType().isFixedLengthVector()) && |
--- |
6649 |
N1.getValueType().isFixedLengthVector()) && |
--- |
| 6650 |
"BUILD_VECTOR used for scalable vectors"); |
--- |
6650 |
"BUILD_VECTOR used for scalable vectors"); |
--- |
| 6651 |
unsigned Index = |
--- |
6651 |
unsigned Index = |
--- |
| 6652 |
N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0; |
0 |
6652 |
N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0; |
0 |
| 6653 |
SDValue Elt = N1.getOperand(Index); |
0 |
6653 |
SDValue Elt = N1.getOperand(Index); |
0 |
| 6654 |
|
--- |
6654 |
|
--- |
| 6655 |
if (VT != Elt.getValueType()) |
0 |
6655 |
if (VT != Elt.getValueType()) |
0 |
| 6656 |
// If the vector element type is not legal, the BUILD_VECTOR operands |
--- |
6656 |
// If the vector element type is not legal, the BUILD_VECTOR operands |
--- |
| 6657 |
// are promoted and implicitly truncated, and the result implicitly |
--- |
6657 |
// are promoted and implicitly truncated, and the result implicitly |
--- |
| 6658 |
// extended. Make that explicit here. |
--- |
6658 |
// extended. Make that explicit here. |
--- |
| 6659 |
Elt = getAnyExtOrTrunc(Elt, DL, VT); |
0 |
6659 |
Elt = getAnyExtOrTrunc(Elt, DL, VT); |
0 |
| 6660 |
|
--- |
6660 |
|
--- |
| 6661 |
return Elt; |
0 |
6661 |
return Elt; |
0 |
| 6662 |
} |
--- |
6662 |
} |
--- |
| 6663 |
|
--- |
6663 |
|
--- |
| 6664 |
// EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector |
--- |
6664 |
// EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector |
--- |
| 6665 |
// operations are lowered to scalars. |
--- |
6665 |
// operations are lowered to scalars. |
--- |
| 6666 |
if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { |
0 |
6666 |
if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { |
0 |
| 6667 |
// If the indices are the same, return the inserted element else |
--- |
6667 |
// If the indices are the same, return the inserted element else |
--- |
| 6668 |
// if the indices are known different, extract the element from |
--- |
6668 |
// if the indices are known different, extract the element from |
--- |
| 6669 |
// the original vector. |
--- |
6669 |
// the original vector. |
--- |
| 6670 |
SDValue N1Op2 = N1.getOperand(2); |
0 |
6670 |
SDValue N1Op2 = N1.getOperand(2); |
0 |
| 6671 |
ConstantSDNode *N1Op2C = dyn_cast(N1Op2); |
0 |
6671 |
ConstantSDNode *N1Op2C = dyn_cast(N1Op2); |
0 |
| 6672 |
|
--- |
6672 |
|
--- |
| 6673 |
if (N1Op2C && N2C) { |
0 |
6673 |
if (N1Op2C && N2C) { |
0 |
| 6674 |
if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { |
0 |
6674 |
if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { |
0 |
| 6675 |
if (VT == N1.getOperand(1).getValueType()) |
0 |
6675 |
if (VT == N1.getOperand(1).getValueType()) |
0 |
| 6676 |
return N1.getOperand(1); |
0 |
6676 |
return N1.getOperand(1); |
0 |
| 6677 |
if (VT.isFloatingPoint()) { |
0 |
6677 |
if (VT.isFloatingPoint()) { |
0 |
| 6678 |
assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits()); |
0 |
6678 |
assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits()); |
0 |
| 6679 |
return getFPExtendOrRound(N1.getOperand(1), DL, VT); |
0 |
6679 |
return getFPExtendOrRound(N1.getOperand(1), DL, VT); |
0 |
| 6680 |
} |
--- |
6680 |
} |
--- |
| 6681 |
return getSExtOrTrunc(N1.getOperand(1), DL, VT); |
0 |
6681 |
return getSExtOrTrunc(N1.getOperand(1), DL, VT); |
0 |
| 6682 |
} |
--- |
6682 |
} |
--- |
| 6683 |
return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); |
0 |
6683 |
return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); |
0 |
| 6684 |
} |
--- |
6684 |
} |
--- |
| 6685 |
} |
--- |
6685 |
} |
--- |
| 6686 |
|
--- |
6686 |
|
--- |
| 6687 |
// EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed |
--- |
6687 |
// EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed |
--- |
| 6688 |
// when vector types are scalarized and v1iX is legal. |
--- |
6688 |
// when vector types are scalarized and v1iX is legal. |
--- |
| 6689 |
// vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx). |
--- |
6689 |
// vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx). |
--- |
| 6690 |
// Here we are completely ignoring the extract element index (N2), |
--- |
6690 |
// Here we are completely ignoring the extract element index (N2), |
--- |
| 6691 |
// which is fine for fixed width vectors, since any index other than 0 |
--- |
6691 |
// which is fine for fixed width vectors, since any index other than 0 |
--- |
| 6692 |
// is undefined anyway. However, this cannot be ignored for scalable |
--- |
6692 |
// is undefined anyway. However, this cannot be ignored for scalable |
--- |
| 6693 |
// vectors - in theory we could support this, but we don't want to do this |
--- |
6693 |
// vectors - in theory we could support this, but we don't want to do this |
--- |
| 6694 |
// without a profitability check. |
--- |
6694 |
// without a profitability check. |
--- |
| 6695 |
if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && |
0 |
6695 |
if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && |
0 |
| 6696 |
N1.getValueType().isFixedLengthVector() && |
0 |
6696 |
N1.getValueType().isFixedLengthVector() && |
0 |
| 6697 |
N1.getValueType().getVectorNumElements() == 1) { |
0 |
6697 |
N1.getValueType().getVectorNumElements() == 1) { |
0 |
| 6698 |
return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), |
0 |
6698 |
return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), |
0 |
| 6699 |
N1.getOperand(1)); |
0 |
6699 |
N1.getOperand(1)); |
0 |
| 6700 |
} |
--- |
6700 |
} |
--- |
| 6701 |
break; |
0 |
6701 |
break; |
0 |
| 6702 |
case ISD::EXTRACT_ELEMENT: |
0 |
6702 |
case ISD::EXTRACT_ELEMENT: |
0 |
| 6703 |
assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!"); |
0 |
6703 |
assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!"); |
0 |
| 6704 |
assert(!N1.getValueType().isVector() && !VT.isVector() && |
0 |
6704 |
assert(!N1.getValueType().isVector() && !VT.isVector() && |
0 |
| 6705 |
(N1.getValueType().isInteger() == VT.isInteger()) && |
--- |
6705 |
(N1.getValueType().isInteger() == VT.isInteger()) && |
--- |
| 6706 |
N1.getValueType() != VT && |
--- |
6706 |
N1.getValueType() != VT && |
--- |
| 6707 |
"Wrong types for EXTRACT_ELEMENT!"); |
--- |
6707 |
"Wrong types for EXTRACT_ELEMENT!"); |
--- |
| 6708 |
|
--- |
6708 |
|
--- |
| 6709 |
// EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding |
--- |
6709 |
// EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding |
--- |
| 6710 |
// 64-bit integers into 32-bit parts. Instead of building the extract of |
--- |
6710 |
// 64-bit integers into 32-bit parts. Instead of building the extract of |
--- |
| 6711 |
// the BUILD_PAIR, only to have legalize rip it apart, just do it now. |
--- |
6711 |
// the BUILD_PAIR, only to have legalize rip it apart, just do it now. |
--- |
| 6712 |
if (N1.getOpcode() == ISD::BUILD_PAIR) |
0 |
6712 |
if (N1.getOpcode() == ISD::BUILD_PAIR) |
0 |
| 6713 |
return N1.getOperand(N2C->getZExtValue()); |
0 |
6713 |
return N1.getOperand(N2C->getZExtValue()); |
0 |
| 6714 |
|
--- |
6714 |
|
--- |
| 6715 |
// EXTRACT_ELEMENT of a constant int is also very common. |
--- |
6715 |
// EXTRACT_ELEMENT of a constant int is also very common. |
--- |
| 6716 |
if (N1C) { |
0 |
6716 |
if (N1C) { |
0 |
| 6717 |
unsigned ElementSize = VT.getSizeInBits(); |
0 |
6717 |
unsigned ElementSize = VT.getSizeInBits(); |
0 |
| 6718 |
unsigned Shift = ElementSize * N2C->getZExtValue(); |
0 |
6718 |
unsigned Shift = ElementSize * N2C->getZExtValue(); |
0 |
| 6719 |
const APInt &Val = N1C->getAPIntValue(); |
0 |
6719 |
const APInt &Val = N1C->getAPIntValue(); |
0 |
| 6720 |
return getConstant(Val.extractBits(ElementSize, Shift), DL, VT); |
0 |
6720 |
return getConstant(Val.extractBits(ElementSize, Shift), DL, VT); |
0 |
| 6721 |
} |
--- |
6721 |
} |
--- |
| 6722 |
break; |
0 |
6722 |
break; |
0 |
| 6723 |
case ISD::EXTRACT_SUBVECTOR: { |
0 |
6723 |
case ISD::EXTRACT_SUBVECTOR: { |
0 |
| 6724 |
EVT N1VT = N1.getValueType(); |
0 |
6724 |
EVT N1VT = N1.getValueType(); |
0 |
| 6725 |
assert(VT.isVector() && N1VT.isVector() && |
0 |
6725 |
assert(VT.isVector() && N1VT.isVector() && |
0 |
| 6726 |
"Extract subvector VTs must be vectors!"); |
--- |
6726 |
"Extract subvector VTs must be vectors!"); |
--- |
| 6727 |
assert(VT.getVectorElementType() == N1VT.getVectorElementType() && |
0 |
6727 |
assert(VT.getVectorElementType() == N1VT.getVectorElementType() && |
0 |
| 6728 |
"Extract subvector VTs must have the same element type!"); |
--- |
6728 |
"Extract subvector VTs must have the same element type!"); |
--- |
| 6729 |
assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) && |
0 |
6729 |
assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) && |
0 |
| 6730 |
"Cannot extract a scalable vector from a fixed length vector!"); |
--- |
6730 |
"Cannot extract a scalable vector from a fixed length vector!"); |
--- |
| 6731 |
assert((VT.isScalableVector() != N1VT.isScalableVector() || |
0 |
6731 |
assert((VT.isScalableVector() != N1VT.isScalableVector() || |
0 |
| 6732 |
VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) && |
--- |
6732 |
VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) && |
--- |
| 6733 |
"Extract subvector must be from larger vector to smaller vector!"); |
--- |
6733 |
"Extract subvector must be from larger vector to smaller vector!"); |
--- |
| 6734 |
assert(N2C && "Extract subvector index must be a constant"); |
0 |
6734 |
assert(N2C && "Extract subvector index must be a constant"); |
0 |
| 6735 |
assert((VT.isScalableVector() != N1VT.isScalableVector() || |
0 |
6735 |
assert((VT.isScalableVector() != N1VT.isScalableVector() || |
0 |
| 6736 |
(VT.getVectorMinNumElements() + N2C->getZExtValue()) <= |
--- |
6736 |
(VT.getVectorMinNumElements() + N2C->getZExtValue()) <= |
--- |
| 6737 |
N1VT.getVectorMinNumElements()) && |
--- |
6737 |
N1VT.getVectorMinNumElements()) && |
--- |
| 6738 |
"Extract subvector overflow!"); |
--- |
6738 |
"Extract subvector overflow!"); |
--- |
| 6739 |
assert(N2C->getAPIntValue().getBitWidth() == |
0 |
6739 |
assert(N2C->getAPIntValue().getBitWidth() == |
0 |
| 6740 |
TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() && |
--- |
6740 |
TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() && |
--- |
| 6741 |
"Constant index for EXTRACT_SUBVECTOR has an invalid size"); |
--- |
6741 |
"Constant index for EXTRACT_SUBVECTOR has an invalid size"); |
--- |
| 6742 |
|
--- |
6742 |
|
--- |
| 6743 |
// Trivial extraction. |
--- |
6743 |
// Trivial extraction. |
--- |
| 6744 |
if (VT == N1VT) |
0 |
6744 |
if (VT == N1VT) |
0 |
| 6745 |
return N1; |
0 |
6745 |
return N1; |
0 |
| 6746 |
|
--- |
6746 |
|
--- |
| 6747 |
// EXTRACT_SUBVECTOR of an UNDEF is an UNDEF. |
--- |
6747 |
// EXTRACT_SUBVECTOR of an UNDEF is an UNDEF. |
--- |
| 6748 |
if (N1.isUndef()) |
0 |
6748 |
if (N1.isUndef()) |
0 |
| 6749 |
return getUNDEF(VT); |
0 |
6749 |
return getUNDEF(VT); |
0 |
| 6750 |
|
--- |
6750 |
|
--- |
| 6751 |
// EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of |
--- |
6751 |
// EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of |
--- |
| 6752 |
// the concat have the same type as the extract. |
--- |
6752 |
// the concat have the same type as the extract. |
--- |
| 6753 |
if (N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0 && |
0 |
6753 |
if (N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0 && |
0 |
| 6754 |
VT == N1.getOperand(0).getValueType()) { |
0 |
6754 |
VT == N1.getOperand(0).getValueType()) { |
0 |
| 6755 |
unsigned Factor = VT.getVectorMinNumElements(); |
0 |
6755 |
unsigned Factor = VT.getVectorMinNumElements(); |
0 |
| 6756 |
return N1.getOperand(N2C->getZExtValue() / Factor); |
0 |
6756 |
return N1.getOperand(N2C->getZExtValue() / Factor); |
0 |
| 6757 |
} |
--- |
6757 |
} |
--- |
| 6758 |
|
--- |
6758 |
|
--- |
| 6759 |
// EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created |
--- |
6759 |
// EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created |
--- |
| 6760 |
// during shuffle legalization. |
--- |
6760 |
// during shuffle legalization. |
--- |
| 6761 |
if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) && |
0 |
6761 |
if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) && |
0 |
| 6762 |
VT == N1.getOperand(1).getValueType()) |
0 |
6762 |
VT == N1.getOperand(1).getValueType()) |
0 |
| 6763 |
return N1.getOperand(1); |
0 |
6763 |
return N1.getOperand(1); |
0 |
| 6764 |
break; |
0 |
6764 |
break; |
0 |
| 6765 |
} |
--- |
6765 |
} |
--- |
| 6766 |
} |
--- |
6766 |
} |
--- |
| 6767 |
|
--- |
6767 |
|
--- |
| 6768 |
// Perform trivial constant folding. |
--- |
6768 |
// Perform trivial constant folding. |
--- |
| 6769 |
if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2})) |
5 |
6769 |
if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2})) |
5 |
| 6770 |
return SV; |
0 |
6770 |
return SV; |
0 |
| 6771 |
|
--- |
6771 |
|
--- |
| 6772 |
// Canonicalize an UNDEF to the RHS, even over a constant. |
--- |
6772 |
// Canonicalize an UNDEF to the RHS, even over a constant. |
--- |
| 6773 |
if (N1.isUndef()) { |
5 |
6773 |
if (N1.isUndef()) { |
5 |
| 6774 |
if (TLI->isCommutativeBinOp(Opcode)) { |
0 |
6774 |
if (TLI->isCommutativeBinOp(Opcode)) { |
0 |
| 6775 |
std::swap(N1, N2); |
0 |
6775 |
std::swap(N1, N2); |
0 |
| 6776 |
} else { |
--- |
6776 |
} else { |
--- |
| 6777 |
switch (Opcode) { |
0 |
6777 |
switch (Opcode) { |
0 |
| 6778 |
case ISD::SUB: |
0 |
6778 |
case ISD::SUB: |
0 |
| 6779 |
return getUNDEF(VT); // fold op(undef, arg2) -> undef |
0 |
6779 |
return getUNDEF(VT); // fold op(undef, arg2) -> undef |
0 |
| 6780 |
case ISD::SIGN_EXTEND_INREG: |
0 |
6780 |
case ISD::SIGN_EXTEND_INREG: |
0 |
| 6781 |
case ISD::UDIV: |
--- |
6781 |
case ISD::UDIV: |
--- |
| 6782 |
case ISD::SDIV: |
--- |
6782 |
case ISD::SDIV: |
--- |
| 6783 |
case ISD::UREM: |
--- |
6783 |
case ISD::UREM: |
--- |
| 6784 |
case ISD::SREM: |
--- |
6784 |
case ISD::SREM: |
--- |
| 6785 |
case ISD::SSUBSAT: |
--- |
6785 |
case ISD::SSUBSAT: |
--- |
| 6786 |
case ISD::USUBSAT: |
--- |
6786 |
case ISD::USUBSAT: |
--- |
| 6787 |
return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0 |
0 |
6787 |
return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0 |
0 |
| 6788 |
} |
--- |
6788 |
} |
--- |
| 6789 |
} |
--- |
6789 |
} |
--- |
| 6790 |
} |
--- |
6790 |
} |
--- |
| 6791 |
|
--- |
6791 |
|
--- |
| 6792 |
// Fold a bunch of operators when the RHS is undef. |
--- |
6792 |
// Fold a bunch of operators when the RHS is undef. |
--- |
| 6793 |
if (N2.isUndef()) { |
5 |
6793 |
if (N2.isUndef()) { |
5 |
| 6794 |
switch (Opcode) { |
0 |
6794 |
switch (Opcode) { |
0 |
| 6795 |
case ISD::XOR: |
0 |
6795 |
case ISD::XOR: |
0 |
| 6796 |
if (N1.isUndef()) |
0 |
6796 |
if (N1.isUndef()) |
0 |
| 6797 |
// Handle undef ^ undef -> 0 special case. This is a common |
--- |
6797 |
// Handle undef ^ undef -> 0 special case. This is a common |
--- |
| 6798 |
// idiom (misuse). |
--- |
6798 |
// idiom (misuse). |
--- |
| 6799 |
return getConstant(0, DL, VT); |
0 |
6799 |
return getConstant(0, DL, VT); |
0 |
| 6800 |
[[fallthrough]]; |
--- |
6800 |
[[fallthrough]]; |
--- |
| 6801 |
case ISD::ADD: |
--- |
6801 |
case ISD::ADD: |
--- |
| 6802 |
case ISD::SUB: |
--- |
6802 |
case ISD::SUB: |
--- |
| 6803 |
case ISD::UDIV: |
--- |
6803 |
case ISD::UDIV: |
--- |
| 6804 |
case ISD::SDIV: |
--- |
6804 |
case ISD::SDIV: |
--- |
| 6805 |
case ISD::UREM: |
--- |
6805 |
case ISD::UREM: |
--- |
| 6806 |
case ISD::SREM: |
--- |
6806 |
case ISD::SREM: |
--- |
| 6807 |
return getUNDEF(VT); // fold op(arg1, undef) -> undef |
0 |
6807 |
return getUNDEF(VT); // fold op(arg1, undef) -> undef |
0 |
| 6808 |
case ISD::MUL: |
0 |
6808 |
case ISD::MUL: |
0 |
| 6809 |
case ISD::AND: |
--- |
6809 |
case ISD::AND: |
--- |
| 6810 |
case ISD::SSUBSAT: |
--- |
6810 |
case ISD::SSUBSAT: |
--- |
| 6811 |
case ISD::USUBSAT: |
--- |
6811 |
case ISD::USUBSAT: |
--- |
| 6812 |
return getConstant(0, DL, VT); // fold op(arg1, undef) -> 0 |
0 |
6812 |
return getConstant(0, DL, VT); // fold op(arg1, undef) -> 0 |
0 |
| 6813 |
case ISD::OR: |
0 |
6813 |
case ISD::OR: |
0 |
| 6814 |
case ISD::SADDSAT: |
--- |
6814 |
case ISD::SADDSAT: |
--- |
| 6815 |
case ISD::UADDSAT: |
--- |
6815 |
case ISD::UADDSAT: |
--- |
| 6816 |
return getAllOnesConstant(DL, VT); |
0 |
6816 |
return getAllOnesConstant(DL, VT); |
0 |
| 6817 |
} |
--- |
6817 |
} |
--- |
| 6818 |
} |
--- |
6818 |
} |
--- |
| 6819 |
|
--- |
6819 |
|
--- |
| 6820 |
// Memoize this node if possible. |
--- |
6820 |
// Memoize this node if possible. |
--- |
| 6821 |
SDNode *N; |
--- |
6821 |
SDNode *N; |
--- |
| 6822 |
SDVTList VTs = getVTList(VT); |
5 |
6822 |
SDVTList VTs = getVTList(VT); |
5 |
| 6823 |
SDValue Ops[] = {N1, N2}; |
5 |
6823 |
SDValue Ops[] = {N1, N2}; |
5 |
| 6824 |
if (VT != MVT::Glue) { |
5 |
6824 |
if (VT != MVT::Glue) { |
5 |
| 6825 |
FoldingSetNodeID ID; |
5 |
6825 |
FoldingSetNodeID ID; |
5 |
| 6826 |
AddNodeIDNode(ID, Opcode, VTs, Ops); |
5 |
6826 |
AddNodeIDNode(ID, Opcode, VTs, Ops); |
5 |
| 6827 |
void *IP = nullptr; |
5 |
6827 |
void *IP = nullptr; |
5 |
| 6828 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { |
5 |
6828 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { |
5 |
| 6829 |
E->intersectFlagsWith(Flags); |
0 |
6829 |
E->intersectFlagsWith(Flags); |
0 |
| 6830 |
return SDValue(E, 0); |
0 |
6830 |
return SDValue(E, 0); |
0 |
| 6831 |
} |
--- |
6831 |
} |
--- |
| 6832 |
|
--- |
6832 |
|
--- |
| 6833 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
5 |
6833 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
5 |
| 6834 |
N->setFlags(Flags); |
5 |
6834 |
N->setFlags(Flags); |
5 |
| 6835 |
createOperands(N, Ops); |
5 |
6835 |
createOperands(N, Ops); |
5 |
| 6836 |
CSEMap.InsertNode(N, IP); |
5 |
6836 |
CSEMap.InsertNode(N, IP); |
5 |
| 6837 |
} else { |
5 |
6837 |
} else { |
5 |
| 6838 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
0 |
6838 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
0 |
| 6839 |
createOperands(N, Ops); |
0 |
6839 |
createOperands(N, Ops); |
0 |
| 6840 |
} |
--- |
6840 |
} |
--- |
| 6841 |
|
--- |
6841 |
|
--- |
| 6842 |
InsertNode(N); |
5 |
6842 |
InsertNode(N); |
5 |
| 6843 |
SDValue V = SDValue(N, 0); |
5 |
6843 |
SDValue V = SDValue(N, 0); |
5 |
| 6844 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
5 |
6844 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
5 |
| 6845 |
return V; |
5 |
6845 |
return V; |
5 |
| 6846 |
} |
--- |
6846 |
} |
--- |
| 6847 |
|
--- |
6847 |
|
--- |
| 6848 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
4 |
6848 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
4 |
| 6849 |
SDValue N1, SDValue N2, SDValue N3) { |
--- |
6849 |
SDValue N1, SDValue N2, SDValue N3) { |
--- |
| 6850 |
SDNodeFlags Flags; |
4 |
6850 |
SDNodeFlags Flags; |
4 |
| 6851 |
if (Inserter) |
4 |
6851 |
if (Inserter) |
4 |
| 6852 |
Flags = Inserter->getFlags(); |
0 |
6852 |
Flags = Inserter->getFlags(); |
0 |
| 6853 |
return getNode(Opcode, DL, VT, N1, N2, N3, Flags); |
4 |
6853 |
return getNode(Opcode, DL, VT, N1, N2, N3, Flags); |
4 |
| 6854 |
} |
--- |
6854 |
} |
--- |
| 6855 |
|
--- |
6855 |
|
--- |
| 6856 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
6 |
6856 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
6 |
| 6857 |
SDValue N1, SDValue N2, SDValue N3, |
--- |
6857 |
SDValue N1, SDValue N2, SDValue N3, |
--- |
| 6858 |
const SDNodeFlags Flags) { |
--- |
6858 |
const SDNodeFlags Flags) { |
--- |
| 6859 |
assert(N1.getOpcode() != ISD::DELETED_NODE && |
6 |
6859 |
assert(N1.getOpcode() != ISD::DELETED_NODE && |
6 |
| 6860 |
N2.getOpcode() != ISD::DELETED_NODE && |
--- |
6860 |
N2.getOpcode() != ISD::DELETED_NODE && |
--- |
| 6861 |
N3.getOpcode() != ISD::DELETED_NODE && |
--- |
6861 |
N3.getOpcode() != ISD::DELETED_NODE && |
--- |
| 6862 |
"Operand is DELETED_NODE!"); |
--- |
6862 |
"Operand is DELETED_NODE!"); |
--- |
| 6863 |
// Perform various simplifications. |
--- |
6863 |
// Perform various simplifications. |
--- |
| 6864 |
switch (Opcode) { |
6 |
6864 |
switch (Opcode) { |
6 |
| 6865 |
case ISD::FMA: { |
0 |
6865 |
case ISD::FMA: { |
0 |
| 6866 |
assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); |
0 |
6866 |
assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); |
0 |
| 6867 |
assert(N1.getValueType() == VT && N2.getValueType() == VT && |
0 |
6867 |
assert(N1.getValueType() == VT && N2.getValueType() == VT && |
0 |
| 6868 |
N3.getValueType() == VT && "FMA types must match!"); |
--- |
6868 |
N3.getValueType() == VT && "FMA types must match!"); |
--- |
| 6869 |
ConstantFPSDNode *N1CFP = dyn_cast(N1); |
0 |
6869 |
ConstantFPSDNode *N1CFP = dyn_cast(N1); |
0 |
| 6870 |
ConstantFPSDNode *N2CFP = dyn_cast(N2); |
0 |
6870 |
ConstantFPSDNode *N2CFP = dyn_cast(N2); |
0 |
| 6871 |
ConstantFPSDNode *N3CFP = dyn_cast(N3); |
0 |
6871 |
ConstantFPSDNode *N3CFP = dyn_cast(N3); |
0 |
| 6872 |
if (N1CFP && N2CFP && N3CFP) { |
0 |
6872 |
if (N1CFP && N2CFP && N3CFP) { |
0 |
| 6873 |
APFloat V1 = N1CFP->getValueAPF(); |
0 |
6873 |
APFloat V1 = N1CFP->getValueAPF(); |
0 |
| 6874 |
const APFloat &V2 = N2CFP->getValueAPF(); |
0 |
6874 |
const APFloat &V2 = N2CFP->getValueAPF(); |
0 |
| 6875 |
const APFloat &V3 = N3CFP->getValueAPF(); |
0 |
6875 |
const APFloat &V3 = N3CFP->getValueAPF(); |
0 |
| 6876 |
V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven); |
0 |
6876 |
V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven); |
0 |
| 6877 |
return getConstantFP(V1, DL, VT); |
0 |
6877 |
return getConstantFP(V1, DL, VT); |
0 |
| 6878 |
} |
0 |
6878 |
} |
0 |
| 6879 |
break; |
0 |
6879 |
break; |
0 |
| 6880 |
} |
--- |
6880 |
} |
--- |
| 6881 |
case ISD::BUILD_VECTOR: { |
0 |
6881 |
case ISD::BUILD_VECTOR: { |
0 |
| 6882 |
// Attempt to simplify BUILD_VECTOR. |
--- |
6882 |
// Attempt to simplify BUILD_VECTOR. |
--- |
| 6883 |
SDValue Ops[] = {N1, N2, N3}; |
0 |
6883 |
SDValue Ops[] = {N1, N2, N3}; |
0 |
| 6884 |
if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this)) |
0 |
6884 |
if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this)) |
0 |
| 6885 |
return V; |
0 |
6885 |
return V; |
0 |
| 6886 |
break; |
0 |
6886 |
break; |
0 |
| 6887 |
} |
--- |
6887 |
} |
--- |
| 6888 |
case ISD::CONCAT_VECTORS: { |
0 |
6888 |
case ISD::CONCAT_VECTORS: { |
0 |
| 6889 |
SDValue Ops[] = {N1, N2, N3}; |
0 |
6889 |
SDValue Ops[] = {N1, N2, N3}; |
0 |
| 6890 |
if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this)) |
0 |
6890 |
if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this)) |
0 |
| 6891 |
return V; |
0 |
6891 |
return V; |
0 |
| 6892 |
break; |
0 |
6892 |
break; |
0 |
| 6893 |
} |
--- |
6893 |
} |
--- |
| 6894 |
case ISD::SETCC: { |
3 |
6894 |
case ISD::SETCC: { |
3 |
| 6895 |
assert(VT.isInteger() && "SETCC result type must be an integer!"); |
3 |
6895 |
assert(VT.isInteger() && "SETCC result type must be an integer!"); |
3 |
| 6896 |
assert(N1.getValueType() == N2.getValueType() && |
3 |
6896 |
assert(N1.getValueType() == N2.getValueType() && |
3 |
| 6897 |
"SETCC operands must have the same type!"); |
--- |
6897 |
"SETCC operands must have the same type!"); |
--- |
| 6898 |
assert(VT.isVector() == N1.getValueType().isVector() && |
3 |
6898 |
assert(VT.isVector() == N1.getValueType().isVector() && |
3 |
| 6899 |
"SETCC type should be vector iff the operand type is vector!"); |
--- |
6899 |
"SETCC type should be vector iff the operand type is vector!"); |
--- |
| 6900 |
assert((!VT.isVector() || VT.getVectorElementCount() == |
3 |
6900 |
assert((!VT.isVector() || VT.getVectorElementCount() == |
3 |
| 6901 |
N1.getValueType().getVectorElementCount()) && |
--- |
6901 |
N1.getValueType().getVectorElementCount()) && |
--- |
| 6902 |
"SETCC vector element counts must match!"); |
--- |
6902 |
"SETCC vector element counts must match!"); |
--- |
| 6903 |
// Use FoldSetCC to simplify SETCC's. |
--- |
6903 |
// Use FoldSetCC to simplify SETCC's. |
--- |
| 6904 |
if (SDValue V = FoldSetCC(VT, N1, N2, cast(N3)->get(), DL)) |
3 |
6904 |
if (SDValue V = FoldSetCC(VT, N1, N2, cast(N3)->get(), DL)) |
3 |
| 6905 |
return V; |
0 |
6905 |
return V; |
0 |
| 6906 |
// Vector constant folding. |
--- |
6906 |
// Vector constant folding. |
--- |
| 6907 |
SDValue Ops[] = {N1, N2, N3}; |
3 |
6907 |
SDValue Ops[] = {N1, N2, N3}; |
3 |
| 6908 |
if (SDValue V = FoldConstantArithmetic(Opcode, DL, VT, Ops)) { |
3 |
6908 |
if (SDValue V = FoldConstantArithmetic(Opcode, DL, VT, Ops)) { |
3 |
| 6909 |
NewSDValueDbgMsg(V, "New node vector constant folding: ", this); |
0 |
6909 |
NewSDValueDbgMsg(V, "New node vector constant folding: ", this); |
0 |
| 6910 |
return V; |
0 |
6910 |
return V; |
0 |
| 6911 |
} |
--- |
6911 |
} |
--- |
| 6912 |
break; |
3 |
6912 |
break; |
3 |
| 6913 |
} |
--- |
6913 |
} |
--- |
| 6914 |
case ISD::SELECT: |
0 |
6914 |
case ISD::SELECT: |
0 |
| 6915 |
case ISD::VSELECT: |
--- |
6915 |
case ISD::VSELECT: |
--- |
| 6916 |
if (SDValue V = simplifySelect(N1, N2, N3)) |
0 |
6916 |
if (SDValue V = simplifySelect(N1, N2, N3)) |
0 |
| 6917 |
return V; |
0 |
6917 |
return V; |
0 |
| 6918 |
break; |
0 |
6918 |
break; |
0 |
| 6919 |
case ISD::VECTOR_SHUFFLE: |
0 |
6919 |
case ISD::VECTOR_SHUFFLE: |
0 |
| 6920 |
llvm_unreachable("should use getVectorShuffle constructor!"); |
0 |
6920 |
llvm_unreachable("should use getVectorShuffle constructor!"); |
0 |
| 6921 |
case ISD::VECTOR_SPLICE: { |
0 |
6921 |
case ISD::VECTOR_SPLICE: { |
0 |
| 6922 |
if (cast(N3)->isZero()) |
0 |
6922 |
if (cast(N3)->isZero()) |
0 |
| 6923 |
return N1; |
0 |
6923 |
return N1; |
0 |
| 6924 |
break; |
0 |
6924 |
break; |
0 |
| 6925 |
} |
--- |
6925 |
} |
--- |
| 6926 |
case ISD::INSERT_VECTOR_ELT: { |
0 |
6926 |
case ISD::INSERT_VECTOR_ELT: { |
0 |
| 6927 |
ConstantSDNode *N3C = dyn_cast(N3); |
0 |
6927 |
ConstantSDNode *N3C = dyn_cast(N3); |
0 |
| 6928 |
// INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except |
--- |
6928 |
// INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except |
--- |
| 6929 |
// for scalable vectors where we will generate appropriate code to |
--- |
6929 |
// for scalable vectors where we will generate appropriate code to |
--- |
| 6930 |
// deal with out-of-bounds cases correctly. |
--- |
6930 |
// deal with out-of-bounds cases correctly. |
--- |
| 6931 |
if (N3C && N1.getValueType().isFixedLengthVector() && |
0 |
6931 |
if (N3C && N1.getValueType().isFixedLengthVector() && |
0 |
| 6932 |
N3C->getZExtValue() >= N1.getValueType().getVectorNumElements()) |
0 |
6932 |
N3C->getZExtValue() >= N1.getValueType().getVectorNumElements()) |
0 |
| 6933 |
return getUNDEF(VT); |
0 |
6933 |
return getUNDEF(VT); |
0 |
| 6934 |
|
--- |
6934 |
|
--- |
| 6935 |
// Undefined index can be assumed out-of-bounds, so that's UNDEF too. |
--- |
6935 |
// Undefined index can be assumed out-of-bounds, so that's UNDEF too. |
--- |
| 6936 |
if (N3.isUndef()) |
0 |
6936 |
if (N3.isUndef()) |
0 |
| 6937 |
return getUNDEF(VT); |
0 |
6937 |
return getUNDEF(VT); |
0 |
| 6938 |
|
--- |
6938 |
|
--- |
| 6939 |
// If the inserted element is an UNDEF, just use the input vector. |
--- |
6939 |
// If the inserted element is an UNDEF, just use the input vector. |
--- |
| 6940 |
if (N2.isUndef()) |
0 |
6940 |
if (N2.isUndef()) |
0 |
| 6941 |
return N1; |
0 |
6941 |
return N1; |
0 |
| 6942 |
|
--- |
6942 |
|
--- |
| 6943 |
break; |
0 |
6943 |
break; |
0 |
| 6944 |
} |
--- |
6944 |
} |
--- |
| 6945 |
case ISD::INSERT_SUBVECTOR: { |
0 |
6945 |
case ISD::INSERT_SUBVECTOR: { |
0 |
| 6946 |
// Inserting undef into undef is still undef. |
--- |
6946 |
// Inserting undef into undef is still undef. |
--- |
| 6947 |
if (N1.isUndef() && N2.isUndef()) |
0 |
6947 |
if (N1.isUndef() && N2.isUndef()) |
0 |
| 6948 |
return getUNDEF(VT); |
0 |
6948 |
return getUNDEF(VT); |
0 |
| 6949 |
|
--- |
6949 |
|
--- |
| 6950 |
EVT N2VT = N2.getValueType(); |
0 |
6950 |
EVT N2VT = N2.getValueType(); |
0 |
| 6951 |
assert(VT == N1.getValueType() && |
0 |
6951 |
assert(VT == N1.getValueType() && |
0 |
| 6952 |
"Dest and insert subvector source types must match!"); |
--- |
6952 |
"Dest and insert subvector source types must match!"); |
--- |
| 6953 |
assert(VT.isVector() && N2VT.isVector() && |
0 |
6953 |
assert(VT.isVector() && N2VT.isVector() && |
0 |
| 6954 |
"Insert subvector VTs must be vectors!"); |
--- |
6954 |
"Insert subvector VTs must be vectors!"); |
--- |
| 6955 |
assert(VT.getVectorElementType() == N2VT.getVectorElementType() && |
0 |
6955 |
assert(VT.getVectorElementType() == N2VT.getVectorElementType() && |
0 |
| 6956 |
"Insert subvector VTs must have the same element type!"); |
--- |
6956 |
"Insert subvector VTs must have the same element type!"); |
--- |
| 6957 |
assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) && |
0 |
6957 |
assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) && |
0 |
| 6958 |
"Cannot insert a scalable vector into a fixed length vector!"); |
--- |
6958 |
"Cannot insert a scalable vector into a fixed length vector!"); |
--- |
| 6959 |
assert((VT.isScalableVector() != N2VT.isScalableVector() || |
0 |
6959 |
assert((VT.isScalableVector() != N2VT.isScalableVector() || |
0 |
| 6960 |
VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) && |
--- |
6960 |
VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) && |
--- |
| 6961 |
"Insert subvector must be from smaller vector to larger vector!"); |
--- |
6961 |
"Insert subvector must be from smaller vector to larger vector!"); |
--- |
| 6962 |
assert(isa(N3) && |
0 |
6962 |
assert(isa(N3) && |
0 |
| 6963 |
"Insert subvector index must be constant"); |
--- |
6963 |
"Insert subvector index must be constant"); |
--- |
| 6964 |
assert((VT.isScalableVector() != N2VT.isScalableVector() || |
0 |
6964 |
assert((VT.isScalableVector() != N2VT.isScalableVector() || |
0 |
| 6965 |
(N2VT.getVectorMinNumElements() + |
--- |
6965 |
(N2VT.getVectorMinNumElements() + |
--- |
| 6966 |
cast(N3)->getZExtValue()) <= |
--- |
6966 |
cast(N3)->getZExtValue()) <= |
--- |
| 6967 |
VT.getVectorMinNumElements()) && |
--- |
6967 |
VT.getVectorMinNumElements()) && |
--- |
| 6968 |
"Insert subvector overflow!"); |
--- |
6968 |
"Insert subvector overflow!"); |
--- |
| 6969 |
assert(cast(N3)->getAPIntValue().getBitWidth() == |
0 |
6969 |
assert(cast(N3)->getAPIntValue().getBitWidth() == |
0 |
| 6970 |
TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() && |
--- |
6970 |
TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() && |
--- |
| 6971 |
"Constant index for INSERT_SUBVECTOR has an invalid size"); |
--- |
6971 |
"Constant index for INSERT_SUBVECTOR has an invalid size"); |
--- |
| 6972 |
|
--- |
6972 |
|
--- |
| 6973 |
// Trivial insertion. |
--- |
6973 |
// Trivial insertion. |
--- |
| 6974 |
if (VT == N2VT) |
0 |
6974 |
if (VT == N2VT) |
0 |
| 6975 |
return N2; |
0 |
6975 |
return N2; |
0 |
| 6976 |
|
--- |
6976 |
|
--- |
| 6977 |
// If this is an insert of an extracted vector into an undef vector, we |
--- |
6977 |
// If this is an insert of an extracted vector into an undef vector, we |
--- |
| 6978 |
// can just use the input to the extract. |
--- |
6978 |
// can just use the input to the extract. |
--- |
| 6979 |
if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR && |
0 |
6979 |
if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR && |
0 |
| 6980 |
N2.getOperand(1) == N3 && N2.getOperand(0).getValueType() == VT) |
0 |
6980 |
N2.getOperand(1) == N3 && N2.getOperand(0).getValueType() == VT) |
0 |
| 6981 |
return N2.getOperand(0); |
0 |
6981 |
return N2.getOperand(0); |
0 |
| 6982 |
break; |
0 |
6982 |
break; |
0 |
| 6983 |
} |
--- |
6983 |
} |
--- |
| 6984 |
case ISD::BITCAST: |
0 |
6984 |
case ISD::BITCAST: |
0 |
| 6985 |
// Fold bit_convert nodes from a type to themselves. |
--- |
6985 |
// Fold bit_convert nodes from a type to themselves. |
--- |
| 6986 |
if (N1.getValueType() == VT) |
0 |
6986 |
if (N1.getValueType() == VT) |
0 |
| 6987 |
return N1; |
0 |
6987 |
return N1; |
0 |
| 6988 |
break; |
0 |
6988 |
break; |
0 |
| 6989 |
case ISD::VP_TRUNCATE: |
0 |
6989 |
case ISD::VP_TRUNCATE: |
0 |
| 6990 |
case ISD::VP_SIGN_EXTEND: |
--- |
6990 |
case ISD::VP_SIGN_EXTEND: |
--- |
| 6991 |
case ISD::VP_ZERO_EXTEND: |
--- |
6991 |
case ISD::VP_ZERO_EXTEND: |
--- |
| 6992 |
// Don't create noop casts. |
--- |
6992 |
// Don't create noop casts. |
--- |
| 6993 |
if (N1.getValueType() == VT) |
0 |
6993 |
if (N1.getValueType() == VT) |
0 |
| 6994 |
return N1; |
0 |
6994 |
return N1; |
0 |
| 6995 |
break; |
0 |
6995 |
break; |
0 |
| 6996 |
} |
--- |
6996 |
} |
--- |
| 6997 |
|
--- |
6997 |
|
--- |
| 6998 |
// Memoize node if it doesn't produce a flag. |
--- |
6998 |
// Memoize node if it doesn't produce a flag. |
--- |
| 6999 |
SDNode *N; |
--- |
6999 |
SDNode *N; |
--- |
| 7000 |
SDVTList VTs = getVTList(VT); |
6 |
7000 |
SDVTList VTs = getVTList(VT); |
6 |
| 7001 |
SDValue Ops[] = {N1, N2, N3}; |
6 |
7001 |
SDValue Ops[] = {N1, N2, N3}; |
6 |
| 7002 |
if (VT != MVT::Glue) { |
6 |
7002 |
if (VT != MVT::Glue) { |
6 |
| 7003 |
FoldingSetNodeID ID; |
6 |
7003 |
FoldingSetNodeID ID; |
6 |
| 7004 |
AddNodeIDNode(ID, Opcode, VTs, Ops); |
6 |
7004 |
AddNodeIDNode(ID, Opcode, VTs, Ops); |
6 |
| 7005 |
void *IP = nullptr; |
6 |
7005 |
void *IP = nullptr; |
6 |
| 7006 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { |
6 |
7006 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { |
6 |
| 7007 |
E->intersectFlagsWith(Flags); |
0 |
7007 |
E->intersectFlagsWith(Flags); |
0 |
| 7008 |
return SDValue(E, 0); |
0 |
7008 |
return SDValue(E, 0); |
0 |
| 7009 |
} |
--- |
7009 |
} |
--- |
| 7010 |
|
--- |
7010 |
|
--- |
| 7011 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
6 |
7011 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
6 |
| 7012 |
N->setFlags(Flags); |
6 |
7012 |
N->setFlags(Flags); |
6 |
| 7013 |
createOperands(N, Ops); |
6 |
7013 |
createOperands(N, Ops); |
6 |
| 7014 |
CSEMap.InsertNode(N, IP); |
6 |
7014 |
CSEMap.InsertNode(N, IP); |
6 |
| 7015 |
} else { |
6 |
7015 |
} else { |
6 |
| 7016 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
0 |
7016 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
0 |
| 7017 |
createOperands(N, Ops); |
0 |
7017 |
createOperands(N, Ops); |
0 |
| 7018 |
} |
--- |
7018 |
} |
--- |
| 7019 |
|
--- |
7019 |
|
--- |
| 7020 |
InsertNode(N); |
6 |
7020 |
InsertNode(N); |
6 |
| 7021 |
SDValue V = SDValue(N, 0); |
6 |
7021 |
SDValue V = SDValue(N, 0); |
6 |
| 7022 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
6 |
7022 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
6 |
| 7023 |
return V; |
6 |
7023 |
return V; |
6 |
| 7024 |
} |
--- |
7024 |
} |
--- |
| 7025 |
|
--- |
7025 |
|
--- |
| 7026 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
0 |
7026 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
0 |
| 7027 |
SDValue N1, SDValue N2, SDValue N3, SDValue N4) { |
--- |
7027 |
SDValue N1, SDValue N2, SDValue N3, SDValue N4) { |
--- |
| 7028 |
SDValue Ops[] = { N1, N2, N3, N4 }; |
0 |
7028 |
SDValue Ops[] = { N1, N2, N3, N4 }; |
0 |
| 7029 |
return getNode(Opcode, DL, VT, Ops); |
0 |
7029 |
return getNode(Opcode, DL, VT, Ops); |
0 |
| 7030 |
} |
--- |
7030 |
} |
--- |
| 7031 |
|
--- |
7031 |
|
--- |
| 7032 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
0 |
7032 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
0 |
| 7033 |
SDValue N1, SDValue N2, SDValue N3, SDValue N4, |
--- |
7033 |
SDValue N1, SDValue N2, SDValue N3, SDValue N4, |
--- |
| 7034 |
SDValue N5) { |
--- |
7034 |
SDValue N5) { |
--- |
| 7035 |
SDValue Ops[] = { N1, N2, N3, N4, N5 }; |
0 |
7035 |
SDValue Ops[] = { N1, N2, N3, N4, N5 }; |
0 |
| 7036 |
return getNode(Opcode, DL, VT, Ops); |
0 |
7036 |
return getNode(Opcode, DL, VT, Ops); |
0 |
| 7037 |
} |
--- |
7037 |
} |
--- |
| 7038 |
|
--- |
7038 |
|
--- |
| 7039 |
/// getStackArgumentTokenFactor - Compute a TokenFactor to force all |
--- |
7039 |
/// getStackArgumentTokenFactor - Compute a TokenFactor to force all |
--- |
| 7040 |
/// the incoming stack arguments to be loaded from the stack. |
--- |
7040 |
/// the incoming stack arguments to be loaded from the stack. |
--- |
| 7041 |
SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { |
0 |
7041 |
SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { |
0 |
| 7042 |
SmallVector ArgChains; |
0 |
7042 |
SmallVector ArgChains; |
0 |
| 7043 |
|
--- |
7043 |
|
--- |
| 7044 |
// Include the original chain at the beginning of the list. When this is |
--- |
7044 |
// Include the original chain at the beginning of the list. When this is |
--- |
| 7045 |
// used by target LowerCall hooks, this helps legalize find the |
--- |
7045 |
// used by target LowerCall hooks, this helps legalize find the |
--- |
| 7046 |
// CALLSEQ_BEGIN node. |
--- |
7046 |
// CALLSEQ_BEGIN node. |
--- |
| 7047 |
ArgChains.push_back(Chain); |
0 |
7047 |
ArgChains.push_back(Chain); |
0 |
| 7048 |
|
--- |
7048 |
|
--- |
| 7049 |
// Add a chain value for each stack argument. |
--- |
7049 |
// Add a chain value for each stack argument. |
--- |
| 7050 |
for (SDNode *U : getEntryNode().getNode()->uses()) |
0 |
7050 |
for (SDNode *U : getEntryNode().getNode()->uses()) |
0 |
| 7051 |
if (LoadSDNode *L = dyn_cast(U)) |
0 |
7051 |
if (LoadSDNode *L = dyn_cast(U)) |
0 |
| 7052 |
if (FrameIndexSDNode *FI = dyn_cast(L->getBasePtr())) |
0 |
7052 |
if (FrameIndexSDNode *FI = dyn_cast(L->getBasePtr())) |
0 |
| 7053 |
if (FI->getIndex() < 0) |
0 |
7053 |
if (FI->getIndex() < 0) |
0 |
| 7054 |
ArgChains.push_back(SDValue(L, 1)); |
0 |
7054 |
ArgChains.push_back(SDValue(L, 1)); |
0 |
| 7055 |
|
--- |
7055 |
|
--- |
| 7056 |
// Build a tokenfactor for all the chains. |
--- |
7056 |
// Build a tokenfactor for all the chains. |
--- |
| 7057 |
return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); |
0 |
7057 |
return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); |
0 |
| 7058 |
} |
0 |
7058 |
} |
0 |
| 7059 |
|
--- |
7059 |
|
--- |
| 7060 |
/// getMemsetValue - Vectorized representation of the memset value |
--- |
7060 |
/// getMemsetValue - Vectorized representation of the memset value |
--- |
| 7061 |
/// operand. |
--- |
7061 |
/// operand. |
--- |
| 7062 |
static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, |
0 |
7062 |
static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, |
0 |
| 7063 |
const SDLoc &dl) { |
--- |
7063 |
const SDLoc &dl) { |
--- |
| 7064 |
assert(!Value.isUndef()); |
0 |
7064 |
assert(!Value.isUndef()); |
0 |
| 7065 |
|
--- |
7065 |
|
--- |
| 7066 |
unsigned NumBits = VT.getScalarSizeInBits(); |
0 |
7066 |
unsigned NumBits = VT.getScalarSizeInBits(); |
0 |
| 7067 |
if (ConstantSDNode *C = dyn_cast(Value)) { |
0 |
7067 |
if (ConstantSDNode *C = dyn_cast(Value)) { |
0 |
| 7068 |
assert(C->getAPIntValue().getBitWidth() == 8); |
0 |
7068 |
assert(C->getAPIntValue().getBitWidth() == 8); |
0 |
| 7069 |
APInt Val = APInt::getSplat(NumBits, C->getAPIntValue()); |
0 |
7069 |
APInt Val = APInt::getSplat(NumBits, C->getAPIntValue()); |
0 |
| 7070 |
if (VT.isInteger()) { |
0 |
7070 |
if (VT.isInteger()) { |
0 |
| 7071 |
bool IsOpaque = VT.getSizeInBits() > 64 || |
0 |
7071 |
bool IsOpaque = VT.getSizeInBits() > 64 || |
0 |
| 7072 |
!DAG.getTargetLoweringInfo().isLegalStoreImmediate(C->getSExtValue()); |
0 |
7072 |
!DAG.getTargetLoweringInfo().isLegalStoreImmediate(C->getSExtValue()); |
0 |
| 7073 |
return DAG.getConstant(Val, dl, VT, false, IsOpaque); |
0 |
7073 |
return DAG.getConstant(Val, dl, VT, false, IsOpaque); |
0 |
| 7074 |
} |
--- |
7074 |
} |
--- |
| 7075 |
return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl, |
0 |
7075 |
return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl, |
0 |
| 7076 |
VT); |
0 |
7076 |
VT); |
0 |
| 7077 |
} |
0 |
7077 |
} |
0 |
| 7078 |
|
--- |
7078 |
|
--- |
| 7079 |
assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?"); |
0 |
7079 |
assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?"); |
0 |
| 7080 |
EVT IntVT = VT.getScalarType(); |
0 |
7080 |
EVT IntVT = VT.getScalarType(); |
0 |
| 7081 |
if (!IntVT.isInteger()) |
0 |
7081 |
if (!IntVT.isInteger()) |
0 |
| 7082 |
IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits()); |
0 |
7082 |
IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits()); |
0 |
| 7083 |
|
--- |
7083 |
|
--- |
| 7084 |
Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value); |
0 |
7084 |
Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value); |
0 |
| 7085 |
if (NumBits > 8) { |
0 |
7085 |
if (NumBits > 8) { |
0 |
| 7086 |
// Use a multiplication with 0x010101... to extend the input to the |
--- |
7086 |
// Use a multiplication with 0x010101... to extend the input to the |
--- |
| 7087 |
// required length. |
--- |
7087 |
// required length. |
--- |
| 7088 |
APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01)); |
0 |
7088 |
APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01)); |
0 |
| 7089 |
Value = DAG.getNode(ISD::MUL, dl, IntVT, Value, |
0 |
7089 |
Value = DAG.getNode(ISD::MUL, dl, IntVT, Value, |
0 |
| 7090 |
DAG.getConstant(Magic, dl, IntVT)); |
--- |
7090 |
DAG.getConstant(Magic, dl, IntVT)); |
--- |
| 7091 |
} |
0 |
7091 |
} |
0 |
| 7092 |
|
--- |
7092 |
|
--- |
| 7093 |
if (VT != Value.getValueType() && !VT.isInteger()) |
0 |
7093 |
if (VT != Value.getValueType() && !VT.isInteger()) |
0 |
| 7094 |
Value = DAG.getBitcast(VT.getScalarType(), Value); |
0 |
7094 |
Value = DAG.getBitcast(VT.getScalarType(), Value); |
0 |
| 7095 |
if (VT != Value.getValueType()) |
0 |
7095 |
if (VT != Value.getValueType()) |
0 |
| 7096 |
Value = DAG.getSplatBuildVector(VT, dl, Value); |
0 |
7096 |
Value = DAG.getSplatBuildVector(VT, dl, Value); |
0 |
| 7097 |
|
--- |
7097 |
|
--- |
| 7098 |
return Value; |
0 |
7098 |
return Value; |
0 |
| 7099 |
} |
--- |
7099 |
} |
--- |
| 7100 |
|
--- |
7100 |
|
--- |
| 7101 |
/// getMemsetStringVal - Similar to getMemsetValue. Except this is only |
--- |
7101 |
/// getMemsetStringVal - Similar to getMemsetValue. Except this is only |
--- |
| 7102 |
/// used when a memcpy is turned into a memset when the source is a constant |
--- |
7102 |
/// used when a memcpy is turned into a memset when the source is a constant |
--- |
| 7103 |
/// string ptr. |
--- |
7103 |
/// string ptr. |
--- |
| 7104 |
static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG, |
0 |
7104 |
static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG, |
0 |
| 7105 |
const TargetLowering &TLI, |
--- |
7105 |
const TargetLowering &TLI, |
--- |
| 7106 |
const ConstantDataArraySlice &Slice) { |
--- |
7106 |
const ConstantDataArraySlice &Slice) { |
--- |
| 7107 |
// Handle vector with all elements zero. |
--- |
7107 |
// Handle vector with all elements zero. |
--- |
| 7108 |
if (Slice.Array == nullptr) { |
0 |
7108 |
if (Slice.Array == nullptr) { |
0 |
| 7109 |
if (VT.isInteger()) |
0 |
7109 |
if (VT.isInteger()) |
0 |
| 7110 |
return DAG.getConstant(0, dl, VT); |
0 |
7110 |
return DAG.getConstant(0, dl, VT); |
0 |
| 7111 |
if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128) |
0 |
7111 |
if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128) |
0 |
| 7112 |
return DAG.getConstantFP(0.0, dl, VT); |
0 |
7112 |
return DAG.getConstantFP(0.0, dl, VT); |
0 |
| 7113 |
if (VT.isVector()) { |
0 |
7113 |
if (VT.isVector()) { |
0 |
| 7114 |
unsigned NumElts = VT.getVectorNumElements(); |
0 |
7114 |
unsigned NumElts = VT.getVectorNumElements(); |
0 |
| 7115 |
MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; |
0 |
7115 |
MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; |
0 |
| 7116 |
return DAG.getNode(ISD::BITCAST, dl, VT, |
0 |
7116 |
return DAG.getNode(ISD::BITCAST, dl, VT, |
0 |
| 7117 |
DAG.getConstant(0, dl, |
--- |
7117 |
DAG.getConstant(0, dl, |
--- |
| 7118 |
EVT::getVectorVT(*DAG.getContext(), |
0 |
7118 |
EVT::getVectorVT(*DAG.getContext(), |
0 |
| 7119 |
EltVT, NumElts))); |
0 |
7119 |
EltVT, NumElts))); |
0 |
| 7120 |
} |
--- |
7120 |
} |
--- |
| 7121 |
llvm_unreachable("Expected type!"); |
0 |
7121 |
llvm_unreachable("Expected type!"); |
0 |
| 7122 |
} |
--- |
7122 |
} |
--- |
| 7123 |
|
--- |
7123 |
|
--- |
| 7124 |
assert(!VT.isVector() && "Can't handle vector type here!"); |
0 |
7124 |
assert(!VT.isVector() && "Can't handle vector type here!"); |
0 |
| 7125 |
unsigned NumVTBits = VT.getSizeInBits(); |
0 |
7125 |
unsigned NumVTBits = VT.getSizeInBits(); |
0 |
| 7126 |
unsigned NumVTBytes = NumVTBits / 8; |
0 |
7126 |
unsigned NumVTBytes = NumVTBits / 8; |
0 |
| 7127 |
unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length)); |
0 |
7127 |
unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length)); |
0 |
| 7128 |
|
--- |
7128 |
|
--- |
| 7129 |
APInt Val(NumVTBits, 0); |
0 |
7129 |
APInt Val(NumVTBits, 0); |
0 |
| 7130 |
if (DAG.getDataLayout().isLittleEndian()) { |
0 |
7130 |
if (DAG.getDataLayout().isLittleEndian()) { |
0 |
| 7131 |
for (unsigned i = 0; i != NumBytes; ++i) |
0 |
7131 |
for (unsigned i = 0; i != NumBytes; ++i) |
0 |
| 7132 |
Val |= (uint64_t)(unsigned char)Slice[i] << i*8; |
0 |
7132 |
Val |= (uint64_t)(unsigned char)Slice[i] << i*8; |
0 |
| 7133 |
} else { |
--- |
7133 |
} else { |
--- |
| 7134 |
for (unsigned i = 0; i != NumBytes; ++i) |
0 |
7134 |
for (unsigned i = 0; i != NumBytes; ++i) |
0 |
| 7135 |
Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8; |
0 |
7135 |
Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8; |
0 |
| 7136 |
} |
--- |
7136 |
} |
--- |
| 7137 |
|
--- |
7137 |
|
--- |
| 7138 |
// If the "cost" of materializing the integer immediate is less than the cost |
--- |
7138 |
// If the "cost" of materializing the integer immediate is less than the cost |
--- |
| 7139 |
// of a load, then it is cost effective to turn the load into the immediate. |
--- |
7139 |
// of a load, then it is cost effective to turn the load into the immediate. |
--- |
| 7140 |
Type *Ty = VT.getTypeForEVT(*DAG.getContext()); |
0 |
7140 |
Type *Ty = VT.getTypeForEVT(*DAG.getContext()); |
0 |
| 7141 |
if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty)) |
0 |
7141 |
if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty)) |
0 |
| 7142 |
return DAG.getConstant(Val, dl, VT); |
0 |
7142 |
return DAG.getConstant(Val, dl, VT); |
0 |
| 7143 |
return SDValue(); |
0 |
7143 |
return SDValue(); |
0 |
| 7144 |
} |
0 |
7144 |
} |
0 |
| 7145 |
|
--- |
7145 |
|
--- |
| 7146 |
SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset, |
4 |
7146 |
SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset, |
4 |
| 7147 |
const SDLoc &DL, |
--- |
7147 |
const SDLoc &DL, |
--- |
| 7148 |
const SDNodeFlags Flags) { |
--- |
7148 |
const SDNodeFlags Flags) { |
--- |
| 7149 |
EVT VT = Base.getValueType(); |
4 |
7149 |
EVT VT = Base.getValueType(); |
4 |
| 7150 |
SDValue Index; |
4 |
7150 |
SDValue Index; |
4 |
| 7151 |
|
--- |
7151 |
|
--- |
| 7152 |
if (Offset.isScalable()) |
4 |
7152 |
if (Offset.isScalable()) |
4 |
| 7153 |
Index = getVScale(DL, Base.getValueType(), |
0 |
7153 |
Index = getVScale(DL, Base.getValueType(), |
0 |
| 7154 |
APInt(Base.getValueSizeInBits().getFixedValue(), |
0 |
7154 |
APInt(Base.getValueSizeInBits().getFixedValue(), |
0 |
| 7155 |
Offset.getKnownMinValue())); |
--- |
7155 |
Offset.getKnownMinValue())); |
--- |
| 7156 |
else |
--- |
7156 |
else |
--- |
| 7157 |
Index = getConstant(Offset.getFixedValue(), DL, VT); |
4 |
7157 |
Index = getConstant(Offset.getFixedValue(), DL, VT); |
4 |
| 7158 |
|
--- |
7158 |
|
--- |
| 7159 |
return getMemBasePlusOffset(Base, Index, DL, Flags); |
4 |
7159 |
return getMemBasePlusOffset(Base, Index, DL, Flags); |
4 |
| 7160 |
} |
--- |
7160 |
} |
--- |
| 7161 |
|
--- |
7161 |
|
--- |
| 7162 |
SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset, |
4 |
7162 |
SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset, |
4 |
| 7163 |
const SDLoc &DL, |
--- |
7163 |
const SDLoc &DL, |
--- |
| 7164 |
const SDNodeFlags Flags) { |
--- |
7164 |
const SDNodeFlags Flags) { |
--- |
| 7165 |
assert(Offset.getValueType().isInteger()); |
4 |
7165 |
assert(Offset.getValueType().isInteger()); |
4 |
| 7166 |
EVT BasePtrVT = Ptr.getValueType(); |
4 |
7166 |
EVT BasePtrVT = Ptr.getValueType(); |
4 |
| 7167 |
return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, Flags); |
4 |
7167 |
return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, Flags); |
4 |
| 7168 |
} |
--- |
7168 |
} |
--- |
| 7169 |
|
--- |
7169 |
|
--- |
| 7170 |
/// Returns true if memcpy source is constant data. |
--- |
7170 |
/// Returns true if memcpy source is constant data. |
--- |
| 7171 |
static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) { |
0 |
7171 |
static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) { |
0 |
| 7172 |
uint64_t SrcDelta = 0; |
0 |
7172 |
uint64_t SrcDelta = 0; |
0 |
| 7173 |
GlobalAddressSDNode *G = nullptr; |
0 |
7173 |
GlobalAddressSDNode *G = nullptr; |
0 |
| 7174 |
if (Src.getOpcode() == ISD::GlobalAddress) |
0 |
7174 |
if (Src.getOpcode() == ISD::GlobalAddress) |
0 |
| 7175 |
G = cast(Src); |
0 |
7175 |
G = cast(Src); |
0 |
| 7176 |
else if (Src.getOpcode() == ISD::ADD && |
0 |
7176 |
else if (Src.getOpcode() == ISD::ADD && |
0 |
| 7177 |
Src.getOperand(0).getOpcode() == ISD::GlobalAddress && |
0 |
7177 |
Src.getOperand(0).getOpcode() == ISD::GlobalAddress && |
0 |
| 7178 |
Src.getOperand(1).getOpcode() == ISD::Constant) { |
0 |
7178 |
Src.getOperand(1).getOpcode() == ISD::Constant) { |
0 |
| 7179 |
G = cast(Src.getOperand(0)); |
0 |
7179 |
G = cast(Src.getOperand(0)); |
0 |
| 7180 |
SrcDelta = cast(Src.getOperand(1))->getZExtValue(); |
0 |
7180 |
SrcDelta = cast(Src.getOperand(1))->getZExtValue(); |
0 |
| 7181 |
} |
--- |
7181 |
} |
--- |
| 7182 |
if (!G) |
0 |
7182 |
if (!G) |
0 |
| 7183 |
return false; |
0 |
7183 |
return false; |
0 |
| 7184 |
|
--- |
7184 |
|
--- |
| 7185 |
return getConstantDataArrayInfo(G->getGlobal(), Slice, 8, |
0 |
7185 |
return getConstantDataArrayInfo(G->getGlobal(), Slice, 8, |
0 |
| 7186 |
SrcDelta + G->getOffset()); |
0 |
7186 |
SrcDelta + G->getOffset()); |
0 |
| 7187 |
} |
--- |
7187 |
} |
--- |
| 7188 |
|
--- |
7188 |
|
--- |
| 7189 |
static bool shouldLowerMemFuncForSize(const MachineFunction &MF, |
0 |
7189 |
static bool shouldLowerMemFuncForSize(const MachineFunction &MF, |
0 |
| 7190 |
SelectionDAG &DAG) { |
--- |
7190 |
SelectionDAG &DAG) { |
--- |
| 7191 |
// On Darwin, -Os means optimize for size without hurting performance, so |
--- |
7191 |
// On Darwin, -Os means optimize for size without hurting performance, so |
--- |
| 7192 |
// only really optimize for size when -Oz (MinSize) is used. |
--- |
7192 |
// only really optimize for size when -Oz (MinSize) is used. |
--- |
| 7193 |
if (MF.getTarget().getTargetTriple().isOSDarwin()) |
0 |
7193 |
if (MF.getTarget().getTargetTriple().isOSDarwin()) |
0 |
| 7194 |
return MF.getFunction().hasMinSize(); |
0 |
7194 |
return MF.getFunction().hasMinSize(); |
0 |
| 7195 |
return DAG.shouldOptForSize(); |
0 |
7195 |
return DAG.shouldOptForSize(); |
0 |
| 7196 |
} |
--- |
7196 |
} |
--- |
| 7197 |
|
--- |
7197 |
|
--- |
| 7198 |
static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl, |
0 |
7198 |
static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl, |
0 |
| 7199 |
SmallVector &OutChains, unsigned From, |
--- |
7199 |
SmallVector &OutChains, unsigned From, |
--- |
| 7200 |
unsigned To, SmallVector &OutLoadChains, |
--- |
7200 |
unsigned To, SmallVector &OutLoadChains, |
--- |
| 7201 |
SmallVector &OutStoreChains) { |
--- |
7201 |
SmallVector &OutStoreChains) { |
--- |
| 7202 |
assert(OutLoadChains.size() && "Missing loads in memcpy inlining"); |
0 |
7202 |
assert(OutLoadChains.size() && "Missing loads in memcpy inlining"); |
0 |
| 7203 |
assert(OutStoreChains.size() && "Missing stores in memcpy inlining"); |
0 |
7203 |
assert(OutStoreChains.size() && "Missing stores in memcpy inlining"); |
0 |
| 7204 |
SmallVector GluedLoadChains; |
0 |
7204 |
SmallVector GluedLoadChains; |
0 |
| 7205 |
for (unsigned i = From; i < To; ++i) { |
0 |
7205 |
for (unsigned i = From; i < To; ++i) { |
0 |
| 7206 |
OutChains.push_back(OutLoadChains[i]); |
0 |
7206 |
OutChains.push_back(OutLoadChains[i]); |
0 |
| 7207 |
GluedLoadChains.push_back(OutLoadChains[i]); |
0 |
7207 |
GluedLoadChains.push_back(OutLoadChains[i]); |
0 |
| 7208 |
} |
--- |
7208 |
} |
--- |
| 7209 |
|
--- |
7209 |
|
--- |
| 7210 |
// Chain for all loads. |
--- |
7210 |
// Chain for all loads. |
--- |
| 7211 |
SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, |
0 |
7211 |
SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, |
0 |
| 7212 |
GluedLoadChains); |
--- |
7212 |
GluedLoadChains); |
--- |
| 7213 |
|
--- |
7213 |
|
--- |
| 7214 |
for (unsigned i = From; i < To; ++i) { |
0 |
7214 |
for (unsigned i = From; i < To; ++i) { |
0 |
| 7215 |
StoreSDNode *ST = dyn_cast(OutStoreChains[i]); |
0 |
7215 |
StoreSDNode *ST = dyn_cast(OutStoreChains[i]); |
0 |
| 7216 |
SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(), |
0 |
7216 |
SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(), |
0 |
| 7217 |
ST->getBasePtr(), ST->getMemoryVT(), |
0 |
7217 |
ST->getBasePtr(), ST->getMemoryVT(), |
0 |
| 7218 |
ST->getMemOperand()); |
--- |
7218 |
ST->getMemOperand()); |
--- |
| 7219 |
OutChains.push_back(NewStore); |
0 |
7219 |
OutChains.push_back(NewStore); |
0 |
| 7220 |
} |
--- |
7220 |
} |
--- |
| 7221 |
} |
0 |
7221 |
} |
0 |
| 7222 |
|
--- |
7222 |
|
--- |
| 7223 |
static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, |
0 |
7223 |
static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, |
0 |
| 7224 |
SDValue Chain, SDValue Dst, SDValue Src, |
--- |
7224 |
SDValue Chain, SDValue Dst, SDValue Src, |
--- |
| 7225 |
uint64_t Size, Align Alignment, |
--- |
7225 |
uint64_t Size, Align Alignment, |
--- |
| 7226 |
bool isVol, bool AlwaysInline, |
--- |
7226 |
bool isVol, bool AlwaysInline, |
--- |
| 7227 |
MachinePointerInfo DstPtrInfo, |
--- |
7227 |
MachinePointerInfo DstPtrInfo, |
--- |
| 7228 |
MachinePointerInfo SrcPtrInfo, |
--- |
7228 |
MachinePointerInfo SrcPtrInfo, |
--- |
| 7229 |
const AAMDNodes &AAInfo, AAResults *AA) { |
--- |
7229 |
const AAMDNodes &AAInfo, AAResults *AA) { |
--- |
| 7230 |
// Turn a memcpy of undef to nop. |
--- |
7230 |
// Turn a memcpy of undef to nop. |
--- |
| 7231 |
// FIXME: We need to honor volatile even is Src is undef. |
--- |
7231 |
// FIXME: We need to honor volatile even is Src is undef. |
--- |
| 7232 |
if (Src.isUndef()) |
0 |
7232 |
if (Src.isUndef()) |
0 |
| 7233 |
return Chain; |
0 |
7233 |
return Chain; |
0 |
| 7234 |
|
--- |
7234 |
|
--- |
| 7235 |
// Expand memcpy to a series of load and store ops if the size operand falls |
--- |
7235 |
// Expand memcpy to a series of load and store ops if the size operand falls |
--- |
| 7236 |
// below a certain threshold. |
--- |
7236 |
// below a certain threshold. |
--- |
| 7237 |
// TODO: In the AlwaysInline case, if the size is big then generate a loop |
--- |
7237 |
// TODO: In the AlwaysInline case, if the size is big then generate a loop |
--- |
| 7238 |
// rather than maybe a humongous number of loads and stores. |
--- |
7238 |
// rather than maybe a humongous number of loads and stores. |
--- |
| 7239 |
const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
0 |
7239 |
const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
0 |
| 7240 |
const DataLayout &DL = DAG.getDataLayout(); |
0 |
7240 |
const DataLayout &DL = DAG.getDataLayout(); |
0 |
| 7241 |
LLVMContext &C = *DAG.getContext(); |
0 |
7241 |
LLVMContext &C = *DAG.getContext(); |
0 |
| 7242 |
std::vector MemOps; |
0 |
7242 |
std::vector MemOps; |
0 |
| 7243 |
bool DstAlignCanChange = false; |
0 |
7243 |
bool DstAlignCanChange = false; |
0 |
| 7244 |
MachineFunction &MF = DAG.getMachineFunction(); |
0 |
7244 |
MachineFunction &MF = DAG.getMachineFunction(); |
0 |
| 7245 |
MachineFrameInfo &MFI = MF.getFrameInfo(); |
0 |
7245 |
MachineFrameInfo &MFI = MF.getFrameInfo(); |
0 |
| 7246 |
bool OptSize = shouldLowerMemFuncForSize(MF, DAG); |
0 |
7246 |
bool OptSize = shouldLowerMemFuncForSize(MF, DAG); |
0 |
| 7247 |
FrameIndexSDNode *FI = dyn_cast(Dst); |
0 |
7247 |
FrameIndexSDNode *FI = dyn_cast(Dst); |
0 |
| 7248 |
if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) |
0 |
7248 |
if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) |
0 |
| 7249 |
DstAlignCanChange = true; |
0 |
7249 |
DstAlignCanChange = true; |
0 |
| 7250 |
MaybeAlign SrcAlign = DAG.InferPtrAlign(Src); |
0 |
7250 |
MaybeAlign SrcAlign = DAG.InferPtrAlign(Src); |
0 |
| 7251 |
if (!SrcAlign || Alignment > *SrcAlign) |
0 |
7251 |
if (!SrcAlign || Alignment > *SrcAlign) |
0 |
| 7252 |
SrcAlign = Alignment; |
0 |
7252 |
SrcAlign = Alignment; |
0 |
| 7253 |
assert(SrcAlign && "SrcAlign must be set"); |
0 |
7253 |
assert(SrcAlign && "SrcAlign must be set"); |
0 |
| 7254 |
ConstantDataArraySlice Slice; |
--- |
7254 |
ConstantDataArraySlice Slice; |
--- |
| 7255 |
// If marked as volatile, perform a copy even when marked as constant. |
--- |
7255 |
// If marked as volatile, perform a copy even when marked as constant. |
--- |
| 7256 |
bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice); |
0 |
7256 |
bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice); |
0 |
| 7257 |
bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr; |
0 |
7257 |
bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr; |
0 |
| 7258 |
unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); |
0 |
7258 |
unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); |
0 |
| 7259 |
const MemOp Op = isZeroConstant |
--- |
7259 |
const MemOp Op = isZeroConstant |
--- |
| 7260 |
? MemOp::Set(Size, DstAlignCanChange, Alignment, |
0 |
7260 |
? MemOp::Set(Size, DstAlignCanChange, Alignment, |
0 |
| 7261 |
/*IsZeroMemset*/ true, isVol) |
--- |
7261 |
/*IsZeroMemset*/ true, isVol) |
--- |
| 7262 |
: MemOp::Copy(Size, DstAlignCanChange, Alignment, |
0 |
7262 |
: MemOp::Copy(Size, DstAlignCanChange, Alignment, |
0 |
| 7263 |
*SrcAlign, isVol, CopyFromConstant); |
0 |
7263 |
*SrcAlign, isVol, CopyFromConstant); |
0 |
| 7264 |
if (!TLI.findOptimalMemOpLowering( |
0 |
7264 |
if (!TLI.findOptimalMemOpLowering( |
0 |
| 7265 |
MemOps, Limit, Op, DstPtrInfo.getAddrSpace(), |
--- |
7265 |
MemOps, Limit, Op, DstPtrInfo.getAddrSpace(), |
--- |
| 7266 |
SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes())) |
0 |
7266 |
SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes())) |
0 |
| 7267 |
return SDValue(); |
0 |
7267 |
return SDValue(); |
0 |
| 7268 |
|
--- |
7268 |
|
--- |
| 7269 |
if (DstAlignCanChange) { |
0 |
7269 |
if (DstAlignCanChange) { |
0 |
| 7270 |
Type *Ty = MemOps[0].getTypeForEVT(C); |
0 |
7270 |
Type *Ty = MemOps[0].getTypeForEVT(C); |
0 |
| 7271 |
Align NewAlign = DL.getABITypeAlign(Ty); |
0 |
7271 |
Align NewAlign = DL.getABITypeAlign(Ty); |
0 |
| 7272 |
|
--- |
7272 |
|
--- |
| 7273 |
// Don't promote to an alignment that would require dynamic stack |
--- |
7273 |
// Don't promote to an alignment that would require dynamic stack |
--- |
| 7274 |
// realignment which may conflict with optimizations such as tail call |
--- |
7274 |
// realignment which may conflict with optimizations such as tail call |
--- |
| 7275 |
// optimization. |
--- |
7275 |
// optimization. |
--- |
| 7276 |
const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
0 |
7276 |
const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
0 |
| 7277 |
if (!TRI->hasStackRealignment(MF)) |
0 |
7277 |
if (!TRI->hasStackRealignment(MF)) |
0 |
| 7278 |
while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign)) |
0 |
7278 |
while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign)) |
0 |
| 7279 |
NewAlign = NewAlign.previous(); |
0 |
7279 |
NewAlign = NewAlign.previous(); |
0 |
| 7280 |
|
--- |
7280 |
|
--- |
| 7281 |
if (NewAlign > Alignment) { |
0 |
7281 |
if (NewAlign > Alignment) { |
0 |
| 7282 |
// Give the stack frame object a larger alignment if needed. |
--- |
7282 |
// Give the stack frame object a larger alignment if needed. |
--- |
| 7283 |
if (MFI.getObjectAlign(FI->getIndex()) < NewAlign) |
0 |
7283 |
if (MFI.getObjectAlign(FI->getIndex()) < NewAlign) |
0 |
| 7284 |
MFI.setObjectAlignment(FI->getIndex(), NewAlign); |
0 |
7284 |
MFI.setObjectAlignment(FI->getIndex(), NewAlign); |
0 |
| 7285 |
Alignment = NewAlign; |
0 |
7285 |
Alignment = NewAlign; |
0 |
| 7286 |
} |
--- |
7286 |
} |
--- |
| 7287 |
} |
--- |
7287 |
} |
--- |
| 7288 |
|
--- |
7288 |
|
--- |
| 7289 |
// Prepare AAInfo for loads/stores after lowering this memcpy. |
--- |
7289 |
// Prepare AAInfo for loads/stores after lowering this memcpy. |
--- |
| 7290 |
AAMDNodes NewAAInfo = AAInfo; |
0 |
7290 |
AAMDNodes NewAAInfo = AAInfo; |
0 |
| 7291 |
NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr; |
0 |
7291 |
NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr; |
0 |
| 7292 |
|
--- |
7292 |
|
--- |
| 7293 |
const Value *SrcVal = dyn_cast_if_present(SrcPtrInfo.V); |
0 |
7293 |
const Value *SrcVal = dyn_cast_if_present(SrcPtrInfo.V); |
0 |
| 7294 |
bool isConstant = |
--- |
7294 |
bool isConstant = |
--- |
| 7295 |
AA && SrcVal && |
0 |
7295 |
AA && SrcVal && |
0 |
| 7296 |
AA->pointsToConstantMemory(MemoryLocation(SrcVal, Size, AAInfo)); |
0 |
7296 |
AA->pointsToConstantMemory(MemoryLocation(SrcVal, Size, AAInfo)); |
0 |
| 7297 |
|
--- |
7297 |
|
--- |
| 7298 |
MachineMemOperand::Flags MMOFlags = |
0 |
7298 |
MachineMemOperand::Flags MMOFlags = |
0 |
| 7299 |
isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; |
0 |
7299 |
isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; |
0 |
| 7300 |
SmallVector OutLoadChains; |
0 |
7300 |
SmallVector OutLoadChains; |
0 |
| 7301 |
SmallVector OutStoreChains; |
0 |
7301 |
SmallVector OutStoreChains; |
0 |
| 7302 |
SmallVector OutChains; |
0 |
7302 |
SmallVector OutChains; |
0 |
| 7303 |
unsigned NumMemOps = MemOps.size(); |
0 |
7303 |
unsigned NumMemOps = MemOps.size(); |
0 |
| 7304 |
uint64_t SrcOff = 0, DstOff = 0; |
0 |
7304 |
uint64_t SrcOff = 0, DstOff = 0; |
0 |
| 7305 |
for (unsigned i = 0; i != NumMemOps; ++i) { |
0 |
7305 |
for (unsigned i = 0; i != NumMemOps; ++i) { |
0 |
| 7306 |
EVT VT = MemOps[i]; |
0 |
7306 |
EVT VT = MemOps[i]; |
0 |
| 7307 |
unsigned VTSize = VT.getSizeInBits() / 8; |
0 |
7307 |
unsigned VTSize = VT.getSizeInBits() / 8; |
0 |
| 7308 |
SDValue Value, Store; |
0 |
7308 |
SDValue Value, Store; |
0 |
| 7309 |
|
--- |
7309 |
|
--- |
| 7310 |
if (VTSize > Size) { |
0 |
7310 |
if (VTSize > Size) { |
0 |
| 7311 |
// Issuing an unaligned load / store pair that overlaps with the previous |
--- |
7311 |
// Issuing an unaligned load / store pair that overlaps with the previous |
--- |
| 7312 |
// pair. Adjust the offset accordingly. |
--- |
7312 |
// pair. Adjust the offset accordingly. |
--- |
| 7313 |
assert(i == NumMemOps-1 && i != 0); |
0 |
7313 |
assert(i == NumMemOps-1 && i != 0); |
0 |
| 7314 |
SrcOff -= VTSize - Size; |
0 |
7314 |
SrcOff -= VTSize - Size; |
0 |
| 7315 |
DstOff -= VTSize - Size; |
0 |
7315 |
DstOff -= VTSize - Size; |
0 |
| 7316 |
} |
--- |
7316 |
} |
--- |
| 7317 |
|
--- |
7317 |
|
--- |
| 7318 |
if (CopyFromConstant && |
0 |
7318 |
if (CopyFromConstant && |
0 |
| 7319 |
(isZeroConstant || (VT.isInteger() && !VT.isVector()))) { |
0 |
7319 |
(isZeroConstant || (VT.isInteger() && !VT.isVector()))) { |
0 |
| 7320 |
// It's unlikely a store of a vector immediate can be done in a single |
--- |
7320 |
// It's unlikely a store of a vector immediate can be done in a single |
--- |
| 7321 |
// instruction. It would require a load from a constantpool first. |
--- |
7321 |
// instruction. It would require a load from a constantpool first. |
--- |
| 7322 |
// We only handle zero vectors here. |
--- |
7322 |
// We only handle zero vectors here. |
--- |
| 7323 |
// FIXME: Handle other cases where store of vector immediate is done in |
--- |
7323 |
// FIXME: Handle other cases where store of vector immediate is done in |
--- |
| 7324 |
// a single instruction. |
--- |
7324 |
// a single instruction. |
--- |
| 7325 |
ConstantDataArraySlice SubSlice; |
--- |
7325 |
ConstantDataArraySlice SubSlice; |
--- |
| 7326 |
if (SrcOff < Slice.Length) { |
0 |
7326 |
if (SrcOff < Slice.Length) { |
0 |
| 7327 |
SubSlice = Slice; |
0 |
7327 |
SubSlice = Slice; |
0 |
| 7328 |
SubSlice.move(SrcOff); |
0 |
7328 |
SubSlice.move(SrcOff); |
0 |
| 7329 |
} else { |
--- |
7329 |
} else { |
--- |
| 7330 |
// This is an out-of-bounds access and hence UB. Pretend we read zero. |
--- |
7330 |
// This is an out-of-bounds access and hence UB. Pretend we read zero. |
--- |
| 7331 |
SubSlice.Array = nullptr; |
0 |
7331 |
SubSlice.Array = nullptr; |
0 |
| 7332 |
SubSlice.Offset = 0; |
0 |
7332 |
SubSlice.Offset = 0; |
0 |
| 7333 |
SubSlice.Length = VTSize; |
0 |
7333 |
SubSlice.Length = VTSize; |
0 |
| 7334 |
} |
--- |
7334 |
} |
--- |
| 7335 |
Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice); |
0 |
7335 |
Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice); |
0 |
| 7336 |
if (Value.getNode()) { |
0 |
7336 |
if (Value.getNode()) { |
0 |
| 7337 |
Store = DAG.getStore( |
0 |
7337 |
Store = DAG.getStore( |
0 |
| 7338 |
Chain, dl, Value, |
--- |
7338 |
Chain, dl, Value, |
--- |
| 7339 |
DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), |
--- |
7339 |
DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), |
--- |
| 7340 |
DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo); |
--- |
7340 |
DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo); |
--- |
| 7341 |
OutChains.push_back(Store); |
0 |
7341 |
OutChains.push_back(Store); |
0 |
| 7342 |
} |
--- |
7342 |
} |
--- |
| 7343 |
} |
--- |
7343 |
} |
--- |
| 7344 |
|
--- |
7344 |
|
--- |
| 7345 |
if (!Store.getNode()) { |
0 |
7345 |
if (!Store.getNode()) { |
0 |
| 7346 |
// The type might not be legal for the target. This should only happen |
--- |
7346 |
// The type might not be legal for the target. This should only happen |
--- |
| 7347 |
// if the type is smaller than a legal type, as on PPC, so the right |
--- |
7347 |
// if the type is smaller than a legal type, as on PPC, so the right |
--- |
| 7348 |
// thing to do is generate a LoadExt/StoreTrunc pair. These simplify |
--- |
7348 |
// thing to do is generate a LoadExt/StoreTrunc pair. These simplify |
--- |
| 7349 |
// to Load/Store if NVT==VT. |
--- |
7349 |
// to Load/Store if NVT==VT. |
--- |
| 7350 |
// FIXME does the case above also need this? |
--- |
7350 |
// FIXME does the case above also need this? |
--- |
| 7351 |
EVT NVT = TLI.getTypeToTransformTo(C, VT); |
0 |
7351 |
EVT NVT = TLI.getTypeToTransformTo(C, VT); |
0 |
| 7352 |
assert(NVT.bitsGE(VT)); |
0 |
7352 |
assert(NVT.bitsGE(VT)); |
0 |
| 7353 |
|
--- |
7353 |
|
--- |
| 7354 |
bool isDereferenceable = |
--- |
7354 |
bool isDereferenceable = |
--- |
| 7355 |
SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL); |
0 |
7355 |
SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL); |
0 |
| 7356 |
MachineMemOperand::Flags SrcMMOFlags = MMOFlags; |
0 |
7356 |
MachineMemOperand::Flags SrcMMOFlags = MMOFlags; |
0 |
| 7357 |
if (isDereferenceable) |
0 |
7357 |
if (isDereferenceable) |
0 |
| 7358 |
SrcMMOFlags |= MachineMemOperand::MODereferenceable; |
0 |
7358 |
SrcMMOFlags |= MachineMemOperand::MODereferenceable; |
0 |
| 7359 |
if (isConstant) |
0 |
7359 |
if (isConstant) |
0 |
| 7360 |
SrcMMOFlags |= MachineMemOperand::MOInvariant; |
0 |
7360 |
SrcMMOFlags |= MachineMemOperand::MOInvariant; |
0 |
| 7361 |
|
--- |
7361 |
|
--- |
| 7362 |
Value = DAG.getExtLoad( |
0 |
7362 |
Value = DAG.getExtLoad( |
0 |
| 7363 |
ISD::EXTLOAD, dl, NVT, Chain, |
--- |
7363 |
ISD::EXTLOAD, dl, NVT, Chain, |
--- |
| 7364 |
DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), |
--- |
7364 |
DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), |
--- |
| 7365 |
SrcPtrInfo.getWithOffset(SrcOff), VT, |
--- |
7365 |
SrcPtrInfo.getWithOffset(SrcOff), VT, |
--- |
| 7366 |
commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo); |
0 |
7366 |
commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo); |
0 |
| 7367 |
OutLoadChains.push_back(Value.getValue(1)); |
0 |
7367 |
OutLoadChains.push_back(Value.getValue(1)); |
0 |
| 7368 |
|
--- |
7368 |
|
--- |
| 7369 |
Store = DAG.getTruncStore( |
0 |
7369 |
Store = DAG.getTruncStore( |
0 |
| 7370 |
Chain, dl, Value, |
--- |
7370 |
Chain, dl, Value, |
--- |
| 7371 |
DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), |
--- |
7371 |
DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), |
--- |
| 7372 |
DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags, NewAAInfo); |
--- |
7372 |
DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags, NewAAInfo); |
--- |
| 7373 |
OutStoreChains.push_back(Store); |
0 |
7373 |
OutStoreChains.push_back(Store); |
0 |
| 7374 |
} |
--- |
7374 |
} |
--- |
| 7375 |
SrcOff += VTSize; |
0 |
7375 |
SrcOff += VTSize; |
0 |
| 7376 |
DstOff += VTSize; |
0 |
7376 |
DstOff += VTSize; |
0 |
| 7377 |
Size -= VTSize; |
0 |
7377 |
Size -= VTSize; |
0 |
| 7378 |
} |
--- |
7378 |
} |
--- |
| 7379 |
|
--- |
7379 |
|
--- |
| 7380 |
unsigned GluedLdStLimit = MaxLdStGlue == 0 ? |
0 |
7380 |
unsigned GluedLdStLimit = MaxLdStGlue == 0 ? |
0 |
| 7381 |
TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue; |
0 |
7381 |
TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue; |
0 |
| 7382 |
unsigned NumLdStInMemcpy = OutStoreChains.size(); |
0 |
7382 |
unsigned NumLdStInMemcpy = OutStoreChains.size(); |
0 |
| 7383 |
|
--- |
7383 |
|
--- |
| 7384 |
if (NumLdStInMemcpy) { |
0 |
7384 |
if (NumLdStInMemcpy) { |
0 |
| 7385 |
// It may be that memcpy might be converted to memset if it's memcpy |
--- |
7385 |
// It may be that memcpy might be converted to memset if it's memcpy |
--- |
| 7386 |
// of constants. In such a case, we won't have loads and stores, but |
--- |
7386 |
// of constants. In such a case, we won't have loads and stores, but |
--- |
| 7387 |
// just stores. In the absence of loads, there is nothing to gang up. |
--- |
7387 |
// just stores. In the absence of loads, there is nothing to gang up. |
--- |
| 7388 |
if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) { |
0 |
7388 |
if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) { |
0 |
| 7389 |
// If target does not care, just leave as it. |
--- |
7389 |
// If target does not care, just leave as it. |
--- |
| 7390 |
for (unsigned i = 0; i < NumLdStInMemcpy; ++i) { |
0 |
7390 |
for (unsigned i = 0; i < NumLdStInMemcpy; ++i) { |
0 |
| 7391 |
OutChains.push_back(OutLoadChains[i]); |
0 |
7391 |
OutChains.push_back(OutLoadChains[i]); |
0 |
| 7392 |
OutChains.push_back(OutStoreChains[i]); |
0 |
7392 |
OutChains.push_back(OutStoreChains[i]); |
0 |
| 7393 |
} |
--- |
7393 |
} |
--- |
| 7394 |
} else { |
--- |
7394 |
} else { |
--- |
| 7395 |
// Ld/St less than/equal limit set by target. |
--- |
7395 |
// Ld/St less than/equal limit set by target. |
--- |
| 7396 |
if (NumLdStInMemcpy <= GluedLdStLimit) { |
0 |
7396 |
if (NumLdStInMemcpy <= GluedLdStLimit) { |
0 |
| 7397 |
chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0, |
0 |
7397 |
chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0, |
0 |
| 7398 |
NumLdStInMemcpy, OutLoadChains, |
--- |
7398 |
NumLdStInMemcpy, OutLoadChains, |
--- |
| 7399 |
OutStoreChains); |
--- |
7399 |
OutStoreChains); |
--- |
| 7400 |
} else { |
--- |
7400 |
} else { |
--- |
| 7401 |
unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit; |
0 |
7401 |
unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit; |
0 |
| 7402 |
unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit; |
0 |
7402 |
unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit; |
0 |
| 7403 |
unsigned GlueIter = 0; |
0 |
7403 |
unsigned GlueIter = 0; |
0 |
| 7404 |
|
--- |
7404 |
|
--- |
| 7405 |
for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) { |
0 |
7405 |
for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) { |
0 |
| 7406 |
unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit; |
0 |
7406 |
unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit; |
0 |
| 7407 |
unsigned IndexTo = NumLdStInMemcpy - GlueIter; |
0 |
7407 |
unsigned IndexTo = NumLdStInMemcpy - GlueIter; |
0 |
| 7408 |
|
--- |
7408 |
|
--- |
| 7409 |
chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo, |
0 |
7409 |
chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo, |
0 |
| 7410 |
OutLoadChains, OutStoreChains); |
--- |
7410 |
OutLoadChains, OutStoreChains); |
--- |
| 7411 |
GlueIter += GluedLdStLimit; |
0 |
7411 |
GlueIter += GluedLdStLimit; |
0 |
| 7412 |
} |
--- |
7412 |
} |
--- |
| 7413 |
|
--- |
7413 |
|
--- |
| 7414 |
// Residual ld/st. |
--- |
7414 |
// Residual ld/st. |
--- |
| 7415 |
if (RemainingLdStInMemcpy) { |
0 |
7415 |
if (RemainingLdStInMemcpy) { |
0 |
| 7416 |
chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0, |
0 |
7416 |
chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0, |
0 |
| 7417 |
RemainingLdStInMemcpy, OutLoadChains, |
--- |
7417 |
RemainingLdStInMemcpy, OutLoadChains, |
--- |
| 7418 |
OutStoreChains); |
--- |
7418 |
OutStoreChains); |
--- |
| 7419 |
} |
--- |
7419 |
} |
--- |
| 7420 |
} |
--- |
7420 |
} |
--- |
| 7421 |
} |
--- |
7421 |
} |
--- |
| 7422 |
} |
--- |
7422 |
} |
--- |
| 7423 |
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); |
0 |
7423 |
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); |
0 |
| 7424 |
} |
0 |
7424 |
} |
0 |
| 7425 |
|
--- |
7425 |
|
--- |
| 7426 |
static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, |
0 |
7426 |
static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, |
0 |
| 7427 |
SDValue Chain, SDValue Dst, SDValue Src, |
--- |
7427 |
SDValue Chain, SDValue Dst, SDValue Src, |
--- |
| 7428 |
uint64_t Size, Align Alignment, |
--- |
7428 |
uint64_t Size, Align Alignment, |
--- |
| 7429 |
bool isVol, bool AlwaysInline, |
--- |
7429 |
bool isVol, bool AlwaysInline, |
--- |
| 7430 |
MachinePointerInfo DstPtrInfo, |
--- |
7430 |
MachinePointerInfo DstPtrInfo, |
--- |
| 7431 |
MachinePointerInfo SrcPtrInfo, |
--- |
7431 |
MachinePointerInfo SrcPtrInfo, |
--- |
| 7432 |
const AAMDNodes &AAInfo) { |
--- |
7432 |
const AAMDNodes &AAInfo) { |
--- |
| 7433 |
// Turn a memmove of undef to nop. |
--- |
7433 |
// Turn a memmove of undef to nop. |
--- |
| 7434 |
// FIXME: We need to honor volatile even is Src is undef. |
--- |
7434 |
// FIXME: We need to honor volatile even is Src is undef. |
--- |
| 7435 |
if (Src.isUndef()) |
0 |
7435 |
if (Src.isUndef()) |
0 |
| 7436 |
return Chain; |
0 |
7436 |
return Chain; |
0 |
| 7437 |
|
--- |
7437 |
|
--- |
| 7438 |
// Expand memmove to a series of load and store ops if the size operand falls |
--- |
7438 |
// Expand memmove to a series of load and store ops if the size operand falls |
--- |
| 7439 |
// below a certain threshold. |
--- |
7439 |
// below a certain threshold. |
--- |
| 7440 |
const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
0 |
7440 |
const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
0 |
| 7441 |
const DataLayout &DL = DAG.getDataLayout(); |
0 |
7441 |
const DataLayout &DL = DAG.getDataLayout(); |
0 |
| 7442 |
LLVMContext &C = *DAG.getContext(); |
0 |
7442 |
LLVMContext &C = *DAG.getContext(); |
0 |
| 7443 |
std::vector MemOps; |
0 |
7443 |
std::vector MemOps; |
0 |
| 7444 |
bool DstAlignCanChange = false; |
0 |
7444 |
bool DstAlignCanChange = false; |
0 |
| 7445 |
MachineFunction &MF = DAG.getMachineFunction(); |
0 |
7445 |
MachineFunction &MF = DAG.getMachineFunction(); |
0 |
| 7446 |
MachineFrameInfo &MFI = MF.getFrameInfo(); |
0 |
7446 |
MachineFrameInfo &MFI = MF.getFrameInfo(); |
0 |
| 7447 |
bool OptSize = shouldLowerMemFuncForSize(MF, DAG); |
0 |
7447 |
bool OptSize = shouldLowerMemFuncForSize(MF, DAG); |
0 |
| 7448 |
FrameIndexSDNode *FI = dyn_cast(Dst); |
0 |
7448 |
FrameIndexSDNode *FI = dyn_cast(Dst); |
0 |
| 7449 |
if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) |
0 |
7449 |
if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) |
0 |
| 7450 |
DstAlignCanChange = true; |
0 |
7450 |
DstAlignCanChange = true; |
0 |
| 7451 |
MaybeAlign SrcAlign = DAG.InferPtrAlign(Src); |
0 |
7451 |
MaybeAlign SrcAlign = DAG.InferPtrAlign(Src); |
0 |
| 7452 |
if (!SrcAlign || Alignment > *SrcAlign) |
0 |
7452 |
if (!SrcAlign || Alignment > *SrcAlign) |
0 |
| 7453 |
SrcAlign = Alignment; |
0 |
7453 |
SrcAlign = Alignment; |
0 |
| 7454 |
assert(SrcAlign && "SrcAlign must be set"); |
0 |
7454 |
assert(SrcAlign && "SrcAlign must be set"); |
0 |
| 7455 |
unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); |
0 |
7455 |
unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); |
0 |
| 7456 |
if (!TLI.findOptimalMemOpLowering( |
0 |
7456 |
if (!TLI.findOptimalMemOpLowering( |
0 |
| 7457 |
MemOps, Limit, |
--- |
7457 |
MemOps, Limit, |
--- |
| 7458 |
MemOp::Copy(Size, DstAlignCanChange, Alignment, *SrcAlign, |
0 |
7458 |
MemOp::Copy(Size, DstAlignCanChange, Alignment, *SrcAlign, |
0 |
| 7459 |
/*IsVolatile*/ true), |
--- |
7459 |
/*IsVolatile*/ true), |
--- |
| 7460 |
DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(), |
--- |
7460 |
DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(), |
--- |
| 7461 |
MF.getFunction().getAttributes())) |
0 |
7461 |
MF.getFunction().getAttributes())) |
0 |
| 7462 |
return SDValue(); |
0 |
7462 |
return SDValue(); |
0 |
| 7463 |
|
--- |
7463 |
|
--- |
| 7464 |
if (DstAlignCanChange) { |
0 |
7464 |
if (DstAlignCanChange) { |
0 |
| 7465 |
Type *Ty = MemOps[0].getTypeForEVT(C); |
0 |
7465 |
Type *Ty = MemOps[0].getTypeForEVT(C); |
0 |
| 7466 |
Align NewAlign = DL.getABITypeAlign(Ty); |
0 |
7466 |
Align NewAlign = DL.getABITypeAlign(Ty); |
0 |
| 7467 |
|
--- |
7467 |
|
--- |
| 7468 |
// Don't promote to an alignment that would require dynamic stack |
--- |
7468 |
// Don't promote to an alignment that would require dynamic stack |
--- |
| 7469 |
// realignment which may conflict with optimizations such as tail call |
--- |
7469 |
// realignment which may conflict with optimizations such as tail call |
--- |
| 7470 |
// optimization. |
--- |
7470 |
// optimization. |
--- |
| 7471 |
const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
0 |
7471 |
const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
0 |
| 7472 |
if (!TRI->hasStackRealignment(MF)) |
0 |
7472 |
if (!TRI->hasStackRealignment(MF)) |
0 |
| 7473 |
while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign)) |
0 |
7473 |
while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign)) |
0 |
| 7474 |
NewAlign = NewAlign.previous(); |
0 |
7474 |
NewAlign = NewAlign.previous(); |
0 |
| 7475 |
|
--- |
7475 |
|
--- |
| 7476 |
if (NewAlign > Alignment) { |
0 |
7476 |
if (NewAlign > Alignment) { |
0 |
| 7477 |
// Give the stack frame object a larger alignment if needed. |
--- |
7477 |
// Give the stack frame object a larger alignment if needed. |
--- |
| 7478 |
if (MFI.getObjectAlign(FI->getIndex()) < NewAlign) |
0 |
7478 |
if (MFI.getObjectAlign(FI->getIndex()) < NewAlign) |
0 |
| 7479 |
MFI.setObjectAlignment(FI->getIndex(), NewAlign); |
0 |
7479 |
MFI.setObjectAlignment(FI->getIndex(), NewAlign); |
0 |
| 7480 |
Alignment = NewAlign; |
0 |
7480 |
Alignment = NewAlign; |
0 |
| 7481 |
} |
--- |
7481 |
} |
--- |
| 7482 |
} |
--- |
7482 |
} |
--- |
| 7483 |
|
--- |
7483 |
|
--- |
| 7484 |
// Prepare AAInfo for loads/stores after lowering this memmove. |
--- |
7484 |
// Prepare AAInfo for loads/stores after lowering this memmove. |
--- |
| 7485 |
AAMDNodes NewAAInfo = AAInfo; |
0 |
7485 |
AAMDNodes NewAAInfo = AAInfo; |
0 |
| 7486 |
NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr; |
0 |
7486 |
NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr; |
0 |
| 7487 |
|
--- |
7487 |
|
--- |
| 7488 |
MachineMemOperand::Flags MMOFlags = |
0 |
7488 |
MachineMemOperand::Flags MMOFlags = |
0 |
| 7489 |
isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; |
0 |
7489 |
isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; |
0 |
| 7490 |
uint64_t SrcOff = 0, DstOff = 0; |
0 |
7490 |
uint64_t SrcOff = 0, DstOff = 0; |
0 |
| 7491 |
SmallVector LoadValues; |
0 |
7491 |
SmallVector LoadValues; |
0 |
| 7492 |
SmallVector LoadChains; |
0 |
7492 |
SmallVector LoadChains; |
0 |
| 7493 |
SmallVector OutChains; |
0 |
7493 |
SmallVector OutChains; |
0 |
| 7494 |
unsigned NumMemOps = MemOps.size(); |
0 |
7494 |
unsigned NumMemOps = MemOps.size(); |
0 |
| 7495 |
for (unsigned i = 0; i < NumMemOps; i++) { |
0 |
7495 |
for (unsigned i = 0; i < NumMemOps; i++) { |
0 |
| 7496 |
EVT VT = MemOps[i]; |
0 |
7496 |
EVT VT = MemOps[i]; |
0 |
| 7497 |
unsigned VTSize = VT.getSizeInBits() / 8; |
0 |
7497 |
unsigned VTSize = VT.getSizeInBits() / 8; |
0 |
| 7498 |
SDValue Value; |
0 |
7498 |
SDValue Value; |
0 |
| 7499 |
|
--- |
7499 |
|
--- |
| 7500 |
bool isDereferenceable = |
--- |
7500 |
bool isDereferenceable = |
--- |
| 7501 |
SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL); |
0 |
7501 |
SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL); |
0 |
| 7502 |
MachineMemOperand::Flags SrcMMOFlags = MMOFlags; |
0 |
7502 |
MachineMemOperand::Flags SrcMMOFlags = MMOFlags; |
0 |
| 7503 |
if (isDereferenceable) |
0 |
7503 |
if (isDereferenceable) |
0 |
| 7504 |
SrcMMOFlags |= MachineMemOperand::MODereferenceable; |
0 |
7504 |
SrcMMOFlags |= MachineMemOperand::MODereferenceable; |
0 |
| 7505 |
|
--- |
7505 |
|
--- |
| 7506 |
Value = DAG.getLoad( |
0 |
7506 |
Value = DAG.getLoad( |
0 |
| 7507 |
VT, dl, Chain, |
--- |
7507 |
VT, dl, Chain, |
--- |
| 7508 |
DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), |
--- |
7508 |
DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), |
--- |
| 7509 |
SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags, NewAAInfo); |
0 |
7509 |
SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags, NewAAInfo); |
0 |
| 7510 |
LoadValues.push_back(Value); |
0 |
7510 |
LoadValues.push_back(Value); |
0 |
| 7511 |
LoadChains.push_back(Value.getValue(1)); |
0 |
7511 |
LoadChains.push_back(Value.getValue(1)); |
0 |
| 7512 |
SrcOff += VTSize; |
0 |
7512 |
SrcOff += VTSize; |
0 |
| 7513 |
} |
--- |
7513 |
} |
--- |
| 7514 |
Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); |
0 |
7514 |
Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); |
0 |
| 7515 |
OutChains.clear(); |
0 |
7515 |
OutChains.clear(); |
0 |
| 7516 |
for (unsigned i = 0; i < NumMemOps; i++) { |
0 |
7516 |
for (unsigned i = 0; i < NumMemOps; i++) { |
0 |
| 7517 |
EVT VT = MemOps[i]; |
0 |
7517 |
EVT VT = MemOps[i]; |
0 |
| 7518 |
unsigned VTSize = VT.getSizeInBits() / 8; |
0 |
7518 |
unsigned VTSize = VT.getSizeInBits() / 8; |
0 |
| 7519 |
SDValue Store; |
0 |
7519 |
SDValue Store; |
0 |
| 7520 |
|
--- |
7520 |
|
--- |
| 7521 |
Store = DAG.getStore( |
0 |
7521 |
Store = DAG.getStore( |
0 |
| 7522 |
Chain, dl, LoadValues[i], |
0 |
7522 |
Chain, dl, LoadValues[i], |
0 |
| 7523 |
DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), |
--- |
7523 |
DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), |
--- |
| 7524 |
DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo); |
--- |
7524 |
DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo); |
--- |
| 7525 |
OutChains.push_back(Store); |
0 |
7525 |
OutChains.push_back(Store); |
0 |
| 7526 |
DstOff += VTSize; |
0 |
7526 |
DstOff += VTSize; |
0 |
| 7527 |
} |
--- |
7527 |
} |
--- |
| 7528 |
|
--- |
7528 |
|
--- |
| 7529 |
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); |
0 |
7529 |
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); |
0 |
| 7530 |
} |
0 |
7530 |
} |
0 |
| 7531 |
|
--- |
7531 |
|
--- |
| 7532 |
/// Lower the call to 'memset' intrinsic function into a series of store |
--- |
7532 |
/// Lower the call to 'memset' intrinsic function into a series of store |
--- |
| 7533 |
/// operations. |
--- |
7533 |
/// operations. |
--- |
| 7534 |
/// |
--- |
7534 |
/// |
--- |
| 7535 |
/// \param DAG Selection DAG where lowered code is placed. |
--- |
7535 |
/// \param DAG Selection DAG where lowered code is placed. |
--- |
| 7536 |
/// \param dl Link to corresponding IR location. |
--- |
7536 |
/// \param dl Link to corresponding IR location. |
--- |
| 7537 |
/// \param Chain Control flow dependency. |
--- |
7537 |
/// \param Chain Control flow dependency. |
--- |
| 7538 |
/// \param Dst Pointer to destination memory location. |
--- |
7538 |
/// \param Dst Pointer to destination memory location. |
--- |
| 7539 |
/// \param Src Value of byte to write into the memory. |
--- |
7539 |
/// \param Src Value of byte to write into the memory. |
--- |
| 7540 |
/// \param Size Number of bytes to write. |
--- |
7540 |
/// \param Size Number of bytes to write. |
--- |
| 7541 |
/// \param Alignment Alignment of the destination in bytes. |
--- |
7541 |
/// \param Alignment Alignment of the destination in bytes. |
--- |
| 7542 |
/// \param isVol True if destination is volatile. |
--- |
7542 |
/// \param isVol True if destination is volatile. |
--- |
| 7543 |
/// \param AlwaysInline Makes sure no function call is generated. |
--- |
7543 |
/// \param AlwaysInline Makes sure no function call is generated. |
--- |
| 7544 |
/// \param DstPtrInfo IR information on the memory pointer. |
--- |
7544 |
/// \param DstPtrInfo IR information on the memory pointer. |
--- |
| 7545 |
/// \returns New head in the control flow, if lowering was successful, empty |
--- |
7545 |
/// \returns New head in the control flow, if lowering was successful, empty |
--- |
| 7546 |
/// SDValue otherwise. |
--- |
7546 |
/// SDValue otherwise. |
--- |
| 7547 |
/// |
--- |
7547 |
/// |
--- |
| 7548 |
/// The function tries to replace 'llvm.memset' intrinsic with several store |
--- |
7548 |
/// The function tries to replace 'llvm.memset' intrinsic with several store |
--- |
| 7549 |
/// operations and value calculation code. This is usually profitable for small |
--- |
7549 |
/// operations and value calculation code. This is usually profitable for small |
--- |
| 7550 |
/// memory size or when the semantic requires inlining. |
--- |
7550 |
/// memory size or when the semantic requires inlining. |
--- |
| 7551 |
static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, |
0 |
7551 |
static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, |
0 |
| 7552 |
SDValue Chain, SDValue Dst, SDValue Src, |
--- |
7552 |
SDValue Chain, SDValue Dst, SDValue Src, |
--- |
| 7553 |
uint64_t Size, Align Alignment, bool isVol, |
--- |
7553 |
uint64_t Size, Align Alignment, bool isVol, |
--- |
| 7554 |
bool AlwaysInline, MachinePointerInfo DstPtrInfo, |
--- |
7554 |
bool AlwaysInline, MachinePointerInfo DstPtrInfo, |
--- |
| 7555 |
const AAMDNodes &AAInfo) { |
--- |
7555 |
const AAMDNodes &AAInfo) { |
--- |
| 7556 |
// Turn a memset of undef to nop. |
--- |
7556 |
// Turn a memset of undef to nop. |
--- |
| 7557 |
// FIXME: We need to honor volatile even is Src is undef. |
--- |
7557 |
// FIXME: We need to honor volatile even is Src is undef. |
--- |
| 7558 |
if (Src.isUndef()) |
0 |
7558 |
if (Src.isUndef()) |
0 |
| 7559 |
return Chain; |
0 |
7559 |
return Chain; |
0 |
| 7560 |
|
--- |
7560 |
|
--- |
| 7561 |
// Expand memset to a series of load/store ops if the size operand |
--- |
7561 |
// Expand memset to a series of load/store ops if the size operand |
--- |
| 7562 |
// falls below a certain threshold. |
--- |
7562 |
// falls below a certain threshold. |
--- |
| 7563 |
const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
0 |
7563 |
const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
0 |
| 7564 |
std::vector MemOps; |
0 |
7564 |
std::vector MemOps; |
0 |
| 7565 |
bool DstAlignCanChange = false; |
0 |
7565 |
bool DstAlignCanChange = false; |
0 |
| 7566 |
MachineFunction &MF = DAG.getMachineFunction(); |
0 |
7566 |
MachineFunction &MF = DAG.getMachineFunction(); |
0 |
| 7567 |
MachineFrameInfo &MFI = MF.getFrameInfo(); |
0 |
7567 |
MachineFrameInfo &MFI = MF.getFrameInfo(); |
0 |
| 7568 |
bool OptSize = shouldLowerMemFuncForSize(MF, DAG); |
0 |
7568 |
bool OptSize = shouldLowerMemFuncForSize(MF, DAG); |
0 |
| 7569 |
FrameIndexSDNode *FI = dyn_cast(Dst); |
0 |
7569 |
FrameIndexSDNode *FI = dyn_cast(Dst); |
0 |
| 7570 |
if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) |
0 |
7570 |
if (FI && !MFI.isFixedObjectIndex(FI->getIndex())) |
0 |
| 7571 |
DstAlignCanChange = true; |
0 |
7571 |
DstAlignCanChange = true; |
0 |
| 7572 |
bool IsZeroVal = isNullConstant(Src); |
0 |
7572 |
bool IsZeroVal = isNullConstant(Src); |
0 |
| 7573 |
unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize); |
0 |
7573 |
unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize); |
0 |
| 7574 |
|
--- |
7574 |
|
--- |
| 7575 |
if (!TLI.findOptimalMemOpLowering( |
0 |
7575 |
if (!TLI.findOptimalMemOpLowering( |
0 |
| 7576 |
MemOps, Limit, |
--- |
7576 |
MemOps, Limit, |
--- |
| 7577 |
MemOp::Set(Size, DstAlignCanChange, Alignment, IsZeroVal, isVol), |
0 |
7577 |
MemOp::Set(Size, DstAlignCanChange, Alignment, IsZeroVal, isVol), |
0 |
| 7578 |
DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes())) |
0 |
7578 |
DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes())) |
0 |
| 7579 |
return SDValue(); |
0 |
7579 |
return SDValue(); |
0 |
| 7580 |
|
--- |
7580 |
|
--- |
| 7581 |
if (DstAlignCanChange) { |
0 |
7581 |
if (DstAlignCanChange) { |
0 |
| 7582 |
Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); |
0 |
7582 |
Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); |
0 |
| 7583 |
const DataLayout &DL = DAG.getDataLayout(); |
0 |
7583 |
const DataLayout &DL = DAG.getDataLayout(); |
0 |
| 7584 |
Align NewAlign = DL.getABITypeAlign(Ty); |
0 |
7584 |
Align NewAlign = DL.getABITypeAlign(Ty); |
0 |
| 7585 |
|
--- |
7585 |
|
--- |
| 7586 |
// Don't promote to an alignment that would require dynamic stack |
--- |
7586 |
// Don't promote to an alignment that would require dynamic stack |
--- |
| 7587 |
// realignment which may conflict with optimizations such as tail call |
--- |
7587 |
// realignment which may conflict with optimizations such as tail call |
--- |
| 7588 |
// optimization. |
--- |
7588 |
// optimization. |
--- |
| 7589 |
const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
0 |
7589 |
const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
0 |
| 7590 |
if (!TRI->hasStackRealignment(MF)) |
0 |
7590 |
if (!TRI->hasStackRealignment(MF)) |
0 |
| 7591 |
while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign)) |
0 |
7591 |
while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign)) |
0 |
| 7592 |
NewAlign = NewAlign.previous(); |
0 |
7592 |
NewAlign = NewAlign.previous(); |
0 |
| 7593 |
|
--- |
7593 |
|
--- |
| 7594 |
if (NewAlign > Alignment) { |
0 |
7594 |
if (NewAlign > Alignment) { |
0 |
| 7595 |
// Give the stack frame object a larger alignment if needed. |
--- |
7595 |
// Give the stack frame object a larger alignment if needed. |
--- |
| 7596 |
if (MFI.getObjectAlign(FI->getIndex()) < NewAlign) |
0 |
7596 |
if (MFI.getObjectAlign(FI->getIndex()) < NewAlign) |
0 |
| 7597 |
MFI.setObjectAlignment(FI->getIndex(), NewAlign); |
0 |
7597 |
MFI.setObjectAlignment(FI->getIndex(), NewAlign); |
0 |
| 7598 |
Alignment = NewAlign; |
0 |
7598 |
Alignment = NewAlign; |
0 |
| 7599 |
} |
--- |
7599 |
} |
--- |
| 7600 |
} |
--- |
7600 |
} |
--- |
| 7601 |
|
--- |
7601 |
|
--- |
| 7602 |
SmallVector OutChains; |
0 |
7602 |
SmallVector OutChains; |
0 |
| 7603 |
uint64_t DstOff = 0; |
0 |
7603 |
uint64_t DstOff = 0; |
0 |
| 7604 |
unsigned NumMemOps = MemOps.size(); |
0 |
7604 |
unsigned NumMemOps = MemOps.size(); |
0 |
| 7605 |
|
--- |
7605 |
|
--- |
| 7606 |
// Find the largest store and generate the bit pattern for it. |
--- |
7606 |
// Find the largest store and generate the bit pattern for it. |
--- |
| 7607 |
EVT LargestVT = MemOps[0]; |
0 |
7607 |
EVT LargestVT = MemOps[0]; |
0 |
| 7608 |
for (unsigned i = 1; i < NumMemOps; i++) |
0 |
7608 |
for (unsigned i = 1; i < NumMemOps; i++) |
0 |
| 7609 |
if (MemOps[i].bitsGT(LargestVT)) |
0 |
7609 |
if (MemOps[i].bitsGT(LargestVT)) |
0 |
| 7610 |
LargestVT = MemOps[i]; |
0 |
7610 |
LargestVT = MemOps[i]; |
0 |
| 7611 |
SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl); |
0 |
7611 |
SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl); |
0 |
| 7612 |
|
--- |
7612 |
|
--- |
| 7613 |
// Prepare AAInfo for loads/stores after lowering this memset. |
--- |
7613 |
// Prepare AAInfo for loads/stores after lowering this memset. |
--- |
| 7614 |
AAMDNodes NewAAInfo = AAInfo; |
0 |
7614 |
AAMDNodes NewAAInfo = AAInfo; |
0 |
| 7615 |
NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr; |
0 |
7615 |
NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr; |
0 |
| 7616 |
|
--- |
7616 |
|
--- |
| 7617 |
for (unsigned i = 0; i < NumMemOps; i++) { |
0 |
7617 |
for (unsigned i = 0; i < NumMemOps; i++) { |
0 |
| 7618 |
EVT VT = MemOps[i]; |
0 |
7618 |
EVT VT = MemOps[i]; |
0 |
| 7619 |
unsigned VTSize = VT.getSizeInBits() / 8; |
0 |
7619 |
unsigned VTSize = VT.getSizeInBits() / 8; |
0 |
| 7620 |
if (VTSize > Size) { |
0 |
7620 |
if (VTSize > Size) { |
0 |
| 7621 |
// Issuing an unaligned load / store pair that overlaps with the previous |
--- |
7621 |
// Issuing an unaligned load / store pair that overlaps with the previous |
--- |
| 7622 |
// pair. Adjust the offset accordingly. |
--- |
7622 |
// pair. Adjust the offset accordingly. |
--- |
| 7623 |
assert(i == NumMemOps-1 && i != 0); |
0 |
7623 |
assert(i == NumMemOps-1 && i != 0); |
0 |
| 7624 |
DstOff -= VTSize - Size; |
0 |
7624 |
DstOff -= VTSize - Size; |
0 |
| 7625 |
} |
--- |
7625 |
} |
--- |
| 7626 |
|
--- |
7626 |
|
--- |
| 7627 |
// If this store is smaller than the largest store see whether we can get |
--- |
7627 |
// If this store is smaller than the largest store see whether we can get |
--- |
| 7628 |
// the smaller value for free with a truncate. |
--- |
7628 |
// the smaller value for free with a truncate. |
--- |
| 7629 |
SDValue Value = MemSetValue; |
0 |
7629 |
SDValue Value = MemSetValue; |
0 |
| 7630 |
if (VT.bitsLT(LargestVT)) { |
0 |
7630 |
if (VT.bitsLT(LargestVT)) { |
0 |
| 7631 |
if (!LargestVT.isVector() && !VT.isVector() && |
0 |
7631 |
if (!LargestVT.isVector() && !VT.isVector() && |
0 |
| 7632 |
TLI.isTruncateFree(LargestVT, VT)) |
0 |
7632 |
TLI.isTruncateFree(LargestVT, VT)) |
0 |
| 7633 |
Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); |
0 |
7633 |
Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); |
0 |
| 7634 |
else |
--- |
7634 |
else |
--- |
| 7635 |
Value = getMemsetValue(Src, VT, DAG, dl); |
0 |
7635 |
Value = getMemsetValue(Src, VT, DAG, dl); |
0 |
| 7636 |
} |
--- |
7636 |
} |
--- |
| 7637 |
assert(Value.getValueType() == VT && "Value with wrong type."); |
0 |
7637 |
assert(Value.getValueType() == VT && "Value with wrong type."); |
0 |
| 7638 |
SDValue Store = DAG.getStore( |
0 |
7638 |
SDValue Store = DAG.getStore( |
0 |
| 7639 |
Chain, dl, Value, |
--- |
7639 |
Chain, dl, Value, |
--- |
| 7640 |
DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), |
--- |
7640 |
DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), |
--- |
| 7641 |
DstPtrInfo.getWithOffset(DstOff), Alignment, |
--- |
7641 |
DstPtrInfo.getWithOffset(DstOff), Alignment, |
--- |
| 7642 |
isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone, |
--- |
7642 |
isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone, |
--- |
| 7643 |
NewAAInfo); |
--- |
7643 |
NewAAInfo); |
--- |
| 7644 |
OutChains.push_back(Store); |
0 |
7644 |
OutChains.push_back(Store); |
0 |
| 7645 |
DstOff += VT.getSizeInBits() / 8; |
0 |
7645 |
DstOff += VT.getSizeInBits() / 8; |
0 |
| 7646 |
Size -= VTSize; |
0 |
7646 |
Size -= VTSize; |
0 |
| 7647 |
} |
--- |
7647 |
} |
--- |
| 7648 |
|
--- |
7648 |
|
--- |
| 7649 |
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); |
0 |
7649 |
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); |
0 |
| 7650 |
} |
0 |
7650 |
} |
0 |
| 7651 |
|
--- |
7651 |
|
--- |
| 7652 |
static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, |
0 |
7652 |
static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, |
0 |
| 7653 |
unsigned AS) { |
--- |
7653 |
unsigned AS) { |
--- |
| 7654 |
// Lowering memcpy / memset / memmove intrinsics to calls is only valid if all |
--- |
7654 |
// Lowering memcpy / memset / memmove intrinsics to calls is only valid if all |
--- |
| 7655 |
// pointer operands can be losslessly bitcasted to pointers of address space 0 |
--- |
7655 |
// pointer operands can be losslessly bitcasted to pointers of address space 0 |
--- |
| 7656 |
if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(AS, 0)) { |
0 |
7656 |
if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(AS, 0)) { |
0 |
| 7657 |
report_fatal_error("cannot lower memory intrinsic in address space " + |
0 |
7657 |
report_fatal_error("cannot lower memory intrinsic in address space " + |
0 |
| 7658 |
Twine(AS)); |
0 |
7658 |
Twine(AS)); |
0 |
| 7659 |
} |
--- |
7659 |
} |
--- |
| 7660 |
} |
0 |
7660 |
} |
0 |
| 7661 |
|
--- |
7661 |
|
--- |
| 7662 |
SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, |
0 |
7662 |
SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, |
0 |
| 7663 |
SDValue Src, SDValue Size, Align Alignment, |
--- |
7663 |
SDValue Src, SDValue Size, Align Alignment, |
--- |
| 7664 |
bool isVol, bool AlwaysInline, bool isTailCall, |
--- |
7664 |
bool isVol, bool AlwaysInline, bool isTailCall, |
--- |
| 7665 |
MachinePointerInfo DstPtrInfo, |
--- |
7665 |
MachinePointerInfo DstPtrInfo, |
--- |
| 7666 |
MachinePointerInfo SrcPtrInfo, |
--- |
7666 |
MachinePointerInfo SrcPtrInfo, |
--- |
| 7667 |
const AAMDNodes &AAInfo, AAResults *AA) { |
--- |
7667 |
const AAMDNodes &AAInfo, AAResults *AA) { |
--- |
| 7668 |
// Check to see if we should lower the memcpy to loads and stores first. |
--- |
7668 |
// Check to see if we should lower the memcpy to loads and stores first. |
--- |
| 7669 |
// For cases within the target-specified limits, this is the best choice. |
--- |
7669 |
// For cases within the target-specified limits, this is the best choice. |
--- |
| 7670 |
ConstantSDNode *ConstantSize = dyn_cast(Size); |
0 |
7670 |
ConstantSDNode *ConstantSize = dyn_cast(Size); |
0 |
| 7671 |
if (ConstantSize) { |
0 |
7671 |
if (ConstantSize) { |
0 |
| 7672 |
// Memcpy with size zero? Just return the original chain. |
--- |
7672 |
// Memcpy with size zero? Just return the original chain. |
--- |
| 7673 |
if (ConstantSize->isZero()) |
0 |
7673 |
if (ConstantSize->isZero()) |
0 |
| 7674 |
return Chain; |
0 |
7674 |
return Chain; |
0 |
| 7675 |
|
--- |
7675 |
|
--- |
| 7676 |
SDValue Result = getMemcpyLoadsAndStores( |
0 |
7676 |
SDValue Result = getMemcpyLoadsAndStores( |
0 |
| 7677 |
*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, |
--- |
7677 |
*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, |
--- |
| 7678 |
isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, AA); |
--- |
7678 |
isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, AA); |
--- |
| 7679 |
if (Result.getNode()) |
0 |
7679 |
if (Result.getNode()) |
0 |
| 7680 |
return Result; |
0 |
7680 |
return Result; |
0 |
| 7681 |
} |
--- |
7681 |
} |
--- |
| 7682 |
|
--- |
7682 |
|
--- |
| 7683 |
// Then check to see if we should lower the memcpy with target-specific |
--- |
7683 |
// Then check to see if we should lower the memcpy with target-specific |
--- |
| 7684 |
// code. If the target chooses to do this, this is the next best. |
--- |
7684 |
// code. If the target chooses to do this, this is the next best. |
--- |
| 7685 |
if (TSI) { |
0 |
7685 |
if (TSI) { |
0 |
| 7686 |
SDValue Result = TSI->EmitTargetCodeForMemcpy( |
0 |
7686 |
SDValue Result = TSI->EmitTargetCodeForMemcpy( |
0 |
| 7687 |
*this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline, |
--- |
7687 |
*this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline, |
--- |
| 7688 |
DstPtrInfo, SrcPtrInfo); |
--- |
7688 |
DstPtrInfo, SrcPtrInfo); |
--- |
| 7689 |
if (Result.getNode()) |
0 |
7689 |
if (Result.getNode()) |
0 |
| 7690 |
return Result; |
0 |
7690 |
return Result; |
0 |
| 7691 |
} |
--- |
7691 |
} |
--- |
| 7692 |
|
--- |
7692 |
|
--- |
| 7693 |
// If we really need inline code and the target declined to provide it, |
--- |
7693 |
// If we really need inline code and the target declined to provide it, |
--- |
| 7694 |
// use a (potentially long) sequence of loads and stores. |
--- |
7694 |
// use a (potentially long) sequence of loads and stores. |
--- |
| 7695 |
if (AlwaysInline) { |
0 |
7695 |
if (AlwaysInline) { |
0 |
| 7696 |
assert(ConstantSize && "AlwaysInline requires a constant size!"); |
0 |
7696 |
assert(ConstantSize && "AlwaysInline requires a constant size!"); |
0 |
| 7697 |
return getMemcpyLoadsAndStores( |
0 |
7697 |
return getMemcpyLoadsAndStores( |
0 |
| 7698 |
*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, |
--- |
7698 |
*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, |
--- |
| 7699 |
isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, AA); |
0 |
7699 |
isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, AA); |
0 |
| 7700 |
} |
--- |
7700 |
} |
--- |
| 7701 |
|
--- |
7701 |
|
--- |
| 7702 |
checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); |
0 |
7702 |
checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); |
0 |
| 7703 |
checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace()); |
0 |
7703 |
checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace()); |
0 |
| 7704 |
|
--- |
7704 |
|
--- |
| 7705 |
// FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc |
--- |
7705 |
// FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc |
--- |
| 7706 |
// memcpy is not guaranteed to be safe. libc memcpys aren't required to |
--- |
7706 |
// memcpy is not guaranteed to be safe. libc memcpys aren't required to |
--- |
| 7707 |
// respect volatile, so they may do things like read or write memory |
--- |
7707 |
// respect volatile, so they may do things like read or write memory |
--- |
| 7708 |
// beyond the given memory regions. But fixing this isn't easy, and most |
--- |
7708 |
// beyond the given memory regions. But fixing this isn't easy, and most |
--- |
| 7709 |
// people don't care. |
--- |
7709 |
// people don't care. |
--- |
| 7710 |
|
--- |
7710 |
|
--- |
| 7711 |
// Emit a library call. |
--- |
7711 |
// Emit a library call. |
--- |
| 7712 |
TargetLowering::ArgListTy Args; |
0 |
7712 |
TargetLowering::ArgListTy Args; |
0 |
| 7713 |
TargetLowering::ArgListEntry Entry; |
0 |
7713 |
TargetLowering::ArgListEntry Entry; |
0 |
| 7714 |
Entry.Ty = Type::getInt8PtrTy(*getContext()); |
0 |
7714 |
Entry.Ty = Type::getInt8PtrTy(*getContext()); |
0 |
| 7715 |
Entry.Node = Dst; Args.push_back(Entry); |
0 |
7715 |
Entry.Node = Dst; Args.push_back(Entry); |
0 |
| 7716 |
Entry.Node = Src; Args.push_back(Entry); |
0 |
7716 |
Entry.Node = Src; Args.push_back(Entry); |
0 |
| 7717 |
|
--- |
7717 |
|
--- |
| 7718 |
Entry.Ty = getDataLayout().getIntPtrType(*getContext()); |
0 |
7718 |
Entry.Ty = getDataLayout().getIntPtrType(*getContext()); |
0 |
| 7719 |
Entry.Node = Size; Args.push_back(Entry); |
0 |
7719 |
Entry.Node = Size; Args.push_back(Entry); |
0 |
| 7720 |
// FIXME: pass in SDLoc |
--- |
7720 |
// FIXME: pass in SDLoc |
--- |
| 7721 |
TargetLowering::CallLoweringInfo CLI(*this); |
0 |
7721 |
TargetLowering::CallLoweringInfo CLI(*this); |
0 |
| 7722 |
CLI.setDebugLoc(dl) |
0 |
7722 |
CLI.setDebugLoc(dl) |
0 |
| 7723 |
.setChain(Chain) |
0 |
7723 |
.setChain(Chain) |
0 |
| 7724 |
.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY), |
0 |
7724 |
.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY), |
0 |
| 7725 |
Dst.getValueType().getTypeForEVT(*getContext()), |
0 |
7725 |
Dst.getValueType().getTypeForEVT(*getContext()), |
0 |
| 7726 |
getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY), |
0 |
7726 |
getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY), |
0 |
| 7727 |
TLI->getPointerTy(getDataLayout())), |
0 |
7727 |
TLI->getPointerTy(getDataLayout())), |
0 |
| 7728 |
std::move(Args)) |
0 |
7728 |
std::move(Args)) |
0 |
| 7729 |
.setDiscardResult() |
0 |
7729 |
.setDiscardResult() |
0 |
| 7730 |
.setTailCall(isTailCall); |
0 |
7730 |
.setTailCall(isTailCall); |
0 |
| 7731 |
|
--- |
7731 |
|
--- |
| 7732 |
std::pair CallResult = TLI->LowerCallTo(CLI); |
0 |
7732 |
std::pair CallResult = TLI->LowerCallTo(CLI); |
0 |
| 7733 |
return CallResult.second; |
0 |
7733 |
return CallResult.second; |
0 |
| 7734 |
} |
0 |
7734 |
} |
0 |
| 7735 |
|
--- |
7735 |
|
--- |
| 7736 |
SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl, |
0 |
7736 |
SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl, |
0 |
| 7737 |
SDValue Dst, SDValue Src, SDValue Size, |
--- |
7737 |
SDValue Dst, SDValue Src, SDValue Size, |
--- |
| 7738 |
Type *SizeTy, unsigned ElemSz, |
--- |
7738 |
Type *SizeTy, unsigned ElemSz, |
--- |
| 7739 |
bool isTailCall, |
--- |
7739 |
bool isTailCall, |
--- |
| 7740 |
MachinePointerInfo DstPtrInfo, |
--- |
7740 |
MachinePointerInfo DstPtrInfo, |
--- |
| 7741 |
MachinePointerInfo SrcPtrInfo) { |
--- |
7741 |
MachinePointerInfo SrcPtrInfo) { |
--- |
| 7742 |
// Emit a library call. |
--- |
7742 |
// Emit a library call. |
--- |
| 7743 |
TargetLowering::ArgListTy Args; |
0 |
7743 |
TargetLowering::ArgListTy Args; |
0 |
| 7744 |
TargetLowering::ArgListEntry Entry; |
0 |
7744 |
TargetLowering::ArgListEntry Entry; |
0 |
| 7745 |
Entry.Ty = getDataLayout().getIntPtrType(*getContext()); |
0 |
7745 |
Entry.Ty = getDataLayout().getIntPtrType(*getContext()); |
0 |
| 7746 |
Entry.Node = Dst; |
0 |
7746 |
Entry.Node = Dst; |
0 |
| 7747 |
Args.push_back(Entry); |
0 |
7747 |
Args.push_back(Entry); |
0 |
| 7748 |
|
--- |
7748 |
|
--- |
| 7749 |
Entry.Node = Src; |
0 |
7749 |
Entry.Node = Src; |
0 |
| 7750 |
Args.push_back(Entry); |
0 |
7750 |
Args.push_back(Entry); |
0 |
| 7751 |
|
--- |
7751 |
|
--- |
| 7752 |
Entry.Ty = SizeTy; |
0 |
7752 |
Entry.Ty = SizeTy; |
0 |
| 7753 |
Entry.Node = Size; |
0 |
7753 |
Entry.Node = Size; |
0 |
| 7754 |
Args.push_back(Entry); |
0 |
7754 |
Args.push_back(Entry); |
0 |
| 7755 |
|
--- |
7755 |
|
--- |
| 7756 |
RTLIB::Libcall LibraryCall = |
--- |
7756 |
RTLIB::Libcall LibraryCall = |
--- |
| 7757 |
RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElemSz); |
0 |
7757 |
RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElemSz); |
0 |
| 7758 |
if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) |
0 |
7758 |
if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) |
0 |
| 7759 |
report_fatal_error("Unsupported element size"); |
0 |
7759 |
report_fatal_error("Unsupported element size"); |
0 |
| 7760 |
|
--- |
7760 |
|
--- |
| 7761 |
TargetLowering::CallLoweringInfo CLI(*this); |
0 |
7761 |
TargetLowering::CallLoweringInfo CLI(*this); |
0 |
| 7762 |
CLI.setDebugLoc(dl) |
0 |
7762 |
CLI.setDebugLoc(dl) |
0 |
| 7763 |
.setChain(Chain) |
0 |
7763 |
.setChain(Chain) |
0 |
| 7764 |
.setLibCallee(TLI->getLibcallCallingConv(LibraryCall), |
0 |
7764 |
.setLibCallee(TLI->getLibcallCallingConv(LibraryCall), |
0 |
| 7765 |
Type::getVoidTy(*getContext()), |
0 |
7765 |
Type::getVoidTy(*getContext()), |
0 |
| 7766 |
getExternalSymbol(TLI->getLibcallName(LibraryCall), |
0 |
7766 |
getExternalSymbol(TLI->getLibcallName(LibraryCall), |
0 |
| 7767 |
TLI->getPointerTy(getDataLayout())), |
0 |
7767 |
TLI->getPointerTy(getDataLayout())), |
0 |
| 7768 |
std::move(Args)) |
0 |
7768 |
std::move(Args)) |
0 |
| 7769 |
.setDiscardResult() |
0 |
7769 |
.setDiscardResult() |
0 |
| 7770 |
.setTailCall(isTailCall); |
0 |
7770 |
.setTailCall(isTailCall); |
0 |
| 7771 |
|
--- |
7771 |
|
--- |
| 7772 |
std::pair CallResult = TLI->LowerCallTo(CLI); |
0 |
7772 |
std::pair CallResult = TLI->LowerCallTo(CLI); |
0 |
| 7773 |
return CallResult.second; |
0 |
7773 |
return CallResult.second; |
0 |
| 7774 |
} |
0 |
7774 |
} |
0 |
| 7775 |
|
--- |
7775 |
|
--- |
| 7776 |
SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, |
0 |
7776 |
SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, |
0 |
| 7777 |
SDValue Src, SDValue Size, Align Alignment, |
--- |
7777 |
SDValue Src, SDValue Size, Align Alignment, |
--- |
| 7778 |
bool isVol, bool isTailCall, |
--- |
7778 |
bool isVol, bool isTailCall, |
--- |
| 7779 |
MachinePointerInfo DstPtrInfo, |
--- |
7779 |
MachinePointerInfo DstPtrInfo, |
--- |
| 7780 |
MachinePointerInfo SrcPtrInfo, |
--- |
7780 |
MachinePointerInfo SrcPtrInfo, |
--- |
| 7781 |
const AAMDNodes &AAInfo, AAResults *AA) { |
--- |
7781 |
const AAMDNodes &AAInfo, AAResults *AA) { |
--- |
| 7782 |
// Check to see if we should lower the memmove to loads and stores first. |
--- |
7782 |
// Check to see if we should lower the memmove to loads and stores first. |
--- |
| 7783 |
// For cases within the target-specified limits, this is the best choice. |
--- |
7783 |
// For cases within the target-specified limits, this is the best choice. |
--- |
| 7784 |
ConstantSDNode *ConstantSize = dyn_cast(Size); |
0 |
7784 |
ConstantSDNode *ConstantSize = dyn_cast(Size); |
0 |
| 7785 |
if (ConstantSize) { |
0 |
7785 |
if (ConstantSize) { |
0 |
| 7786 |
// Memmove with size zero? Just return the original chain. |
--- |
7786 |
// Memmove with size zero? Just return the original chain. |
--- |
| 7787 |
if (ConstantSize->isZero()) |
0 |
7787 |
if (ConstantSize->isZero()) |
0 |
| 7788 |
return Chain; |
0 |
7788 |
return Chain; |
0 |
| 7789 |
|
--- |
7789 |
|
--- |
| 7790 |
SDValue Result = getMemmoveLoadsAndStores( |
0 |
7790 |
SDValue Result = getMemmoveLoadsAndStores( |
0 |
| 7791 |
*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, |
--- |
7791 |
*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, |
--- |
| 7792 |
isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo); |
--- |
7792 |
isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo); |
--- |
| 7793 |
if (Result.getNode()) |
0 |
7793 |
if (Result.getNode()) |
0 |
| 7794 |
return Result; |
0 |
7794 |
return Result; |
0 |
| 7795 |
} |
--- |
7795 |
} |
--- |
| 7796 |
|
--- |
7796 |
|
--- |
| 7797 |
// Then check to see if we should lower the memmove with target-specific |
--- |
7797 |
// Then check to see if we should lower the memmove with target-specific |
--- |
| 7798 |
// code. If the target chooses to do this, this is the next best. |
--- |
7798 |
// code. If the target chooses to do this, this is the next best. |
--- |
| 7799 |
if (TSI) { |
0 |
7799 |
if (TSI) { |
0 |
| 7800 |
SDValue Result = |
--- |
7800 |
SDValue Result = |
--- |
| 7801 |
TSI->EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, |
0 |
7801 |
TSI->EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, |
0 |
| 7802 |
Alignment, isVol, DstPtrInfo, SrcPtrInfo); |
--- |
7802 |
Alignment, isVol, DstPtrInfo, SrcPtrInfo); |
--- |
| 7803 |
if (Result.getNode()) |
0 |
7803 |
if (Result.getNode()) |
0 |
| 7804 |
return Result; |
0 |
7804 |
return Result; |
0 |
| 7805 |
} |
--- |
7805 |
} |
--- |
| 7806 |
|
--- |
7806 |
|
--- |
| 7807 |
checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); |
0 |
7807 |
checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); |
0 |
| 7808 |
checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace()); |
0 |
7808 |
checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace()); |
0 |
| 7809 |
|
--- |
7809 |
|
--- |
| 7810 |
// FIXME: If the memmove is volatile, lowering it to plain libc memmove may |
--- |
7810 |
// FIXME: If the memmove is volatile, lowering it to plain libc memmove may |
--- |
| 7811 |
// not be safe. See memcpy above for more details. |
--- |
7811 |
// not be safe. See memcpy above for more details. |
--- |
| 7812 |
|
--- |
7812 |
|
--- |
| 7813 |
// Emit a library call. |
--- |
7813 |
// Emit a library call. |
--- |
| 7814 |
TargetLowering::ArgListTy Args; |
0 |
7814 |
TargetLowering::ArgListTy Args; |
0 |
| 7815 |
TargetLowering::ArgListEntry Entry; |
0 |
7815 |
TargetLowering::ArgListEntry Entry; |
0 |
| 7816 |
Entry.Ty = Type::getInt8PtrTy(*getContext()); |
0 |
7816 |
Entry.Ty = Type::getInt8PtrTy(*getContext()); |
0 |
| 7817 |
Entry.Node = Dst; Args.push_back(Entry); |
0 |
7817 |
Entry.Node = Dst; Args.push_back(Entry); |
0 |
| 7818 |
Entry.Node = Src; Args.push_back(Entry); |
0 |
7818 |
Entry.Node = Src; Args.push_back(Entry); |
0 |
| 7819 |
|
--- |
7819 |
|
--- |
| 7820 |
Entry.Ty = getDataLayout().getIntPtrType(*getContext()); |
0 |
7820 |
Entry.Ty = getDataLayout().getIntPtrType(*getContext()); |
0 |
| 7821 |
Entry.Node = Size; Args.push_back(Entry); |
0 |
7821 |
Entry.Node = Size; Args.push_back(Entry); |
0 |
| 7822 |
// FIXME: pass in SDLoc |
--- |
7822 |
// FIXME: pass in SDLoc |
--- |
| 7823 |
TargetLowering::CallLoweringInfo CLI(*this); |
0 |
7823 |
TargetLowering::CallLoweringInfo CLI(*this); |
0 |
| 7824 |
CLI.setDebugLoc(dl) |
0 |
7824 |
CLI.setDebugLoc(dl) |
0 |
| 7825 |
.setChain(Chain) |
0 |
7825 |
.setChain(Chain) |
0 |
| 7826 |
.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE), |
0 |
7826 |
.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE), |
0 |
| 7827 |
Dst.getValueType().getTypeForEVT(*getContext()), |
0 |
7827 |
Dst.getValueType().getTypeForEVT(*getContext()), |
0 |
| 7828 |
getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE), |
0 |
7828 |
getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE), |
0 |
| 7829 |
TLI->getPointerTy(getDataLayout())), |
0 |
7829 |
TLI->getPointerTy(getDataLayout())), |
0 |
| 7830 |
std::move(Args)) |
0 |
7830 |
std::move(Args)) |
0 |
| 7831 |
.setDiscardResult() |
0 |
7831 |
.setDiscardResult() |
0 |
| 7832 |
.setTailCall(isTailCall); |
0 |
7832 |
.setTailCall(isTailCall); |
0 |
| 7833 |
|
--- |
7833 |
|
--- |
| 7834 |
std::pair CallResult = TLI->LowerCallTo(CLI); |
0 |
7834 |
std::pair CallResult = TLI->LowerCallTo(CLI); |
0 |
| 7835 |
return CallResult.second; |
0 |
7835 |
return CallResult.second; |
0 |
| 7836 |
} |
0 |
7836 |
} |
0 |
| 7837 |
|
--- |
7837 |
|
--- |
| 7838 |
SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl, |
0 |
7838 |
SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl, |
0 |
| 7839 |
SDValue Dst, SDValue Src, SDValue Size, |
--- |
7839 |
SDValue Dst, SDValue Src, SDValue Size, |
--- |
| 7840 |
Type *SizeTy, unsigned ElemSz, |
--- |
7840 |
Type *SizeTy, unsigned ElemSz, |
--- |
| 7841 |
bool isTailCall, |
--- |
7841 |
bool isTailCall, |
--- |
| 7842 |
MachinePointerInfo DstPtrInfo, |
--- |
7842 |
MachinePointerInfo DstPtrInfo, |
--- |
| 7843 |
MachinePointerInfo SrcPtrInfo) { |
--- |
7843 |
MachinePointerInfo SrcPtrInfo) { |
--- |
| 7844 |
// Emit a library call. |
--- |
7844 |
// Emit a library call. |
--- |
| 7845 |
TargetLowering::ArgListTy Args; |
0 |
7845 |
TargetLowering::ArgListTy Args; |
0 |
| 7846 |
TargetLowering::ArgListEntry Entry; |
0 |
7846 |
TargetLowering::ArgListEntry Entry; |
0 |
| 7847 |
Entry.Ty = getDataLayout().getIntPtrType(*getContext()); |
0 |
7847 |
Entry.Ty = getDataLayout().getIntPtrType(*getContext()); |
0 |
| 7848 |
Entry.Node = Dst; |
0 |
7848 |
Entry.Node = Dst; |
0 |
| 7849 |
Args.push_back(Entry); |
0 |
7849 |
Args.push_back(Entry); |
0 |
| 7850 |
|
--- |
7850 |
|
--- |
| 7851 |
Entry.Node = Src; |
0 |
7851 |
Entry.Node = Src; |
0 |
| 7852 |
Args.push_back(Entry); |
0 |
7852 |
Args.push_back(Entry); |
0 |
| 7853 |
|
--- |
7853 |
|
--- |
| 7854 |
Entry.Ty = SizeTy; |
0 |
7854 |
Entry.Ty = SizeTy; |
0 |
| 7855 |
Entry.Node = Size; |
0 |
7855 |
Entry.Node = Size; |
0 |
| 7856 |
Args.push_back(Entry); |
0 |
7856 |
Args.push_back(Entry); |
0 |
| 7857 |
|
--- |
7857 |
|
--- |
| 7858 |
RTLIB::Libcall LibraryCall = |
--- |
7858 |
RTLIB::Libcall LibraryCall = |
--- |
| 7859 |
RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElemSz); |
0 |
7859 |
RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElemSz); |
0 |
| 7860 |
if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) |
0 |
7860 |
if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) |
0 |
| 7861 |
report_fatal_error("Unsupported element size"); |
0 |
7861 |
report_fatal_error("Unsupported element size"); |
0 |
| 7862 |
|
--- |
7862 |
|
--- |
| 7863 |
TargetLowering::CallLoweringInfo CLI(*this); |
0 |
7863 |
TargetLowering::CallLoweringInfo CLI(*this); |
0 |
| 7864 |
CLI.setDebugLoc(dl) |
0 |
7864 |
CLI.setDebugLoc(dl) |
0 |
| 7865 |
.setChain(Chain) |
0 |
7865 |
.setChain(Chain) |
0 |
| 7866 |
.setLibCallee(TLI->getLibcallCallingConv(LibraryCall), |
0 |
7866 |
.setLibCallee(TLI->getLibcallCallingConv(LibraryCall), |
0 |
| 7867 |
Type::getVoidTy(*getContext()), |
0 |
7867 |
Type::getVoidTy(*getContext()), |
0 |
| 7868 |
getExternalSymbol(TLI->getLibcallName(LibraryCall), |
0 |
7868 |
getExternalSymbol(TLI->getLibcallName(LibraryCall), |
0 |
| 7869 |
TLI->getPointerTy(getDataLayout())), |
0 |
7869 |
TLI->getPointerTy(getDataLayout())), |
0 |
| 7870 |
std::move(Args)) |
0 |
7870 |
std::move(Args)) |
0 |
| 7871 |
.setDiscardResult() |
0 |
7871 |
.setDiscardResult() |
0 |
| 7872 |
.setTailCall(isTailCall); |
0 |
7872 |
.setTailCall(isTailCall); |
0 |
| 7873 |
|
--- |
7873 |
|
--- |
| 7874 |
std::pair CallResult = TLI->LowerCallTo(CLI); |
0 |
7874 |
std::pair CallResult = TLI->LowerCallTo(CLI); |
0 |
| 7875 |
return CallResult.second; |
0 |
7875 |
return CallResult.second; |
0 |
| 7876 |
} |
0 |
7876 |
} |
0 |
| 7877 |
|
--- |
7877 |
|
--- |
| 7878 |
SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, |
0 |
7878 |
SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, |
0 |
| 7879 |
SDValue Src, SDValue Size, Align Alignment, |
--- |
7879 |
SDValue Src, SDValue Size, Align Alignment, |
--- |
| 7880 |
bool isVol, bool AlwaysInline, bool isTailCall, |
--- |
7880 |
bool isVol, bool AlwaysInline, bool isTailCall, |
--- |
| 7881 |
MachinePointerInfo DstPtrInfo, |
--- |
7881 |
MachinePointerInfo DstPtrInfo, |
--- |
| 7882 |
const AAMDNodes &AAInfo) { |
--- |
7882 |
const AAMDNodes &AAInfo) { |
--- |
| 7883 |
// Check to see if we should lower the memset to stores first. |
--- |
7883 |
// Check to see if we should lower the memset to stores first. |
--- |
| 7884 |
// For cases within the target-specified limits, this is the best choice. |
--- |
7884 |
// For cases within the target-specified limits, this is the best choice. |
--- |
| 7885 |
ConstantSDNode *ConstantSize = dyn_cast(Size); |
0 |
7885 |
ConstantSDNode *ConstantSize = dyn_cast(Size); |
0 |
| 7886 |
if (ConstantSize) { |
0 |
7886 |
if (ConstantSize) { |
0 |
| 7887 |
// Memset with size zero? Just return the original chain. |
--- |
7887 |
// Memset with size zero? Just return the original chain. |
--- |
| 7888 |
if (ConstantSize->isZero()) |
0 |
7888 |
if (ConstantSize->isZero()) |
0 |
| 7889 |
return Chain; |
0 |
7889 |
return Chain; |
0 |
| 7890 |
|
--- |
7890 |
|
--- |
| 7891 |
SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src, |
0 |
7891 |
SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src, |
0 |
| 7892 |
ConstantSize->getZExtValue(), Alignment, |
--- |
7892 |
ConstantSize->getZExtValue(), Alignment, |
--- |
| 7893 |
isVol, false, DstPtrInfo, AAInfo); |
--- |
7893 |
isVol, false, DstPtrInfo, AAInfo); |
--- |
| 7894 |
|
--- |
7894 |
|
--- |
| 7895 |
if (Result.getNode()) |
0 |
7895 |
if (Result.getNode()) |
0 |
| 7896 |
return Result; |
0 |
7896 |
return Result; |
0 |
| 7897 |
} |
--- |
7897 |
} |
--- |
| 7898 |
|
--- |
7898 |
|
--- |
| 7899 |
// Then check to see if we should lower the memset with target-specific |
--- |
7899 |
// Then check to see if we should lower the memset with target-specific |
--- |
| 7900 |
// code. If the target chooses to do this, this is the next best. |
--- |
7900 |
// code. If the target chooses to do this, this is the next best. |
--- |
| 7901 |
if (TSI) { |
0 |
7901 |
if (TSI) { |
0 |
| 7902 |
SDValue Result = TSI->EmitTargetCodeForMemset( |
0 |
7902 |
SDValue Result = TSI->EmitTargetCodeForMemset( |
0 |
| 7903 |
*this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline, DstPtrInfo); |
--- |
7903 |
*this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline, DstPtrInfo); |
--- |
| 7904 |
if (Result.getNode()) |
0 |
7904 |
if (Result.getNode()) |
0 |
| 7905 |
return Result; |
0 |
7905 |
return Result; |
0 |
| 7906 |
} |
--- |
7906 |
} |
--- |
| 7907 |
|
--- |
7907 |
|
--- |
| 7908 |
// If we really need inline code and the target declined to provide it, |
--- |
7908 |
// If we really need inline code and the target declined to provide it, |
--- |
| 7909 |
// use a (potentially long) sequence of loads and stores. |
--- |
7909 |
// use a (potentially long) sequence of loads and stores. |
--- |
| 7910 |
if (AlwaysInline) { |
0 |
7910 |
if (AlwaysInline) { |
0 |
| 7911 |
assert(ConstantSize && "AlwaysInline requires a constant size!"); |
0 |
7911 |
assert(ConstantSize && "AlwaysInline requires a constant size!"); |
0 |
| 7912 |
SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src, |
0 |
7912 |
SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src, |
0 |
| 7913 |
ConstantSize->getZExtValue(), Alignment, |
--- |
7913 |
ConstantSize->getZExtValue(), Alignment, |
--- |
| 7914 |
isVol, true, DstPtrInfo, AAInfo); |
--- |
7914 |
isVol, true, DstPtrInfo, AAInfo); |
--- |
| 7915 |
assert(Result && |
0 |
7915 |
assert(Result && |
0 |
| 7916 |
"getMemsetStores must return a valid sequence when AlwaysInline"); |
--- |
7916 |
"getMemsetStores must return a valid sequence when AlwaysInline"); |
--- |
| 7917 |
return Result; |
0 |
7917 |
return Result; |
0 |
| 7918 |
} |
--- |
7918 |
} |
--- |
| 7919 |
|
--- |
7919 |
|
--- |
| 7920 |
checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); |
0 |
7920 |
checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); |
0 |
| 7921 |
|
--- |
7921 |
|
--- |
| 7922 |
// Emit a library call. |
--- |
7922 |
// Emit a library call. |
--- |
| 7923 |
auto &Ctx = *getContext(); |
0 |
7923 |
auto &Ctx = *getContext(); |
0 |
| 7924 |
const auto& DL = getDataLayout(); |
0 |
7924 |
const auto& DL = getDataLayout(); |
0 |
| 7925 |
|
--- |
7925 |
|
--- |
| 7926 |
TargetLowering::CallLoweringInfo CLI(*this); |
0 |
7926 |
TargetLowering::CallLoweringInfo CLI(*this); |
0 |
| 7927 |
// FIXME: pass in SDLoc |
--- |
7927 |
// FIXME: pass in SDLoc |
--- |
| 7928 |
CLI.setDebugLoc(dl).setChain(Chain); |
0 |
7928 |
CLI.setDebugLoc(dl).setChain(Chain); |
0 |
| 7929 |
|
--- |
7929 |
|
--- |
| 7930 |
ConstantSDNode *ConstantSrc = dyn_cast(Src); |
0 |
7930 |
ConstantSDNode *ConstantSrc = dyn_cast(Src); |
0 |
| 7931 |
const bool SrcIsZero = ConstantSrc && ConstantSrc->isZero(); |
0 |
7931 |
const bool SrcIsZero = ConstantSrc && ConstantSrc->isZero(); |
0 |
| 7932 |
const char *BzeroName = getTargetLoweringInfo().getLibcallName(RTLIB::BZERO); |
0 |
7932 |
const char *BzeroName = getTargetLoweringInfo().getLibcallName(RTLIB::BZERO); |
0 |
| 7933 |
|
--- |
7933 |
|
--- |
| 7934 |
// Helper function to create an Entry from Node and Type. |
--- |
7934 |
// Helper function to create an Entry from Node and Type. |
--- |
| 7935 |
const auto CreateEntry = [](SDValue Node, Type *Ty) { |
0 |
7935 |
const auto CreateEntry = [](SDValue Node, Type *Ty) { |
0 |
| 7936 |
TargetLowering::ArgListEntry Entry; |
0 |
7936 |
TargetLowering::ArgListEntry Entry; |
0 |
| 7937 |
Entry.Node = Node; |
0 |
7937 |
Entry.Node = Node; |
0 |
| 7938 |
Entry.Ty = Ty; |
0 |
7938 |
Entry.Ty = Ty; |
0 |
| 7939 |
return Entry; |
0 |
7939 |
return Entry; |
0 |
| 7940 |
}; |
--- |
7940 |
}; |
--- |
| 7941 |
|
--- |
7941 |
|
--- |
| 7942 |
// If zeroing out and bzero is present, use it. |
--- |
7942 |
// If zeroing out and bzero is present, use it. |
--- |
| 7943 |
if (SrcIsZero && BzeroName) { |
0 |
7943 |
if (SrcIsZero && BzeroName) { |
0 |
| 7944 |
TargetLowering::ArgListTy Args; |
0 |
7944 |
TargetLowering::ArgListTy Args; |
0 |
| 7945 |
Args.push_back(CreateEntry(Dst, Type::getInt8PtrTy(Ctx))); |
0 |
7945 |
Args.push_back(CreateEntry(Dst, Type::getInt8PtrTy(Ctx))); |
0 |
| 7946 |
Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx))); |
0 |
7946 |
Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx))); |
0 |
| 7947 |
CLI.setLibCallee( |
0 |
7947 |
CLI.setLibCallee( |
0 |
| 7948 |
TLI->getLibcallCallingConv(RTLIB::BZERO), Type::getVoidTy(Ctx), |
0 |
7948 |
TLI->getLibcallCallingConv(RTLIB::BZERO), Type::getVoidTy(Ctx), |
0 |
| 7949 |
getExternalSymbol(BzeroName, TLI->getPointerTy(DL)), std::move(Args)); |
0 |
7949 |
getExternalSymbol(BzeroName, TLI->getPointerTy(DL)), std::move(Args)); |
0 |
| 7950 |
} else { |
0 |
7950 |
} else { |
0 |
| 7951 |
TargetLowering::ArgListTy Args; |
0 |
7951 |
TargetLowering::ArgListTy Args; |
0 |
| 7952 |
Args.push_back(CreateEntry(Dst, Type::getInt8PtrTy(Ctx))); |
0 |
7952 |
Args.push_back(CreateEntry(Dst, Type::getInt8PtrTy(Ctx))); |
0 |
| 7953 |
Args.push_back(CreateEntry(Src, Src.getValueType().getTypeForEVT(Ctx))); |
0 |
7953 |
Args.push_back(CreateEntry(Src, Src.getValueType().getTypeForEVT(Ctx))); |
0 |
| 7954 |
Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx))); |
0 |
7954 |
Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx))); |
0 |
| 7955 |
CLI.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET), |
0 |
7955 |
CLI.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET), |
0 |
| 7956 |
Dst.getValueType().getTypeForEVT(Ctx), |
0 |
7956 |
Dst.getValueType().getTypeForEVT(Ctx), |
0 |
| 7957 |
getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET), |
0 |
7957 |
getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET), |
0 |
| 7958 |
TLI->getPointerTy(DL)), |
0 |
7958 |
TLI->getPointerTy(DL)), |
0 |
| 7959 |
std::move(Args)); |
0 |
7959 |
std::move(Args)); |
0 |
| 7960 |
} |
0 |
7960 |
} |
0 |
| 7961 |
|
--- |
7961 |
|
--- |
| 7962 |
CLI.setDiscardResult().setTailCall(isTailCall); |
0 |
7962 |
CLI.setDiscardResult().setTailCall(isTailCall); |
0 |
| 7963 |
|
--- |
7963 |
|
--- |
| 7964 |
std::pair CallResult = TLI->LowerCallTo(CLI); |
0 |
7964 |
std::pair CallResult = TLI->LowerCallTo(CLI); |
0 |
| 7965 |
return CallResult.second; |
0 |
7965 |
return CallResult.second; |
0 |
| 7966 |
} |
0 |
7966 |
} |
0 |
| 7967 |
|
--- |
7967 |
|
--- |
| 7968 |
SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl, |
0 |
7968 |
SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl, |
0 |
| 7969 |
SDValue Dst, SDValue Value, SDValue Size, |
--- |
7969 |
SDValue Dst, SDValue Value, SDValue Size, |
--- |
| 7970 |
Type *SizeTy, unsigned ElemSz, |
--- |
7970 |
Type *SizeTy, unsigned ElemSz, |
--- |
| 7971 |
bool isTailCall, |
--- |
7971 |
bool isTailCall, |
--- |
| 7972 |
MachinePointerInfo DstPtrInfo) { |
--- |
7972 |
MachinePointerInfo DstPtrInfo) { |
--- |
| 7973 |
// Emit a library call. |
--- |
7973 |
// Emit a library call. |
--- |
| 7974 |
TargetLowering::ArgListTy Args; |
0 |
7974 |
TargetLowering::ArgListTy Args; |
0 |
| 7975 |
TargetLowering::ArgListEntry Entry; |
0 |
7975 |
TargetLowering::ArgListEntry Entry; |
0 |
| 7976 |
Entry.Ty = getDataLayout().getIntPtrType(*getContext()); |
0 |
7976 |
Entry.Ty = getDataLayout().getIntPtrType(*getContext()); |
0 |
| 7977 |
Entry.Node = Dst; |
0 |
7977 |
Entry.Node = Dst; |
0 |
| 7978 |
Args.push_back(Entry); |
0 |
7978 |
Args.push_back(Entry); |
0 |
| 7979 |
|
--- |
7979 |
|
--- |
| 7980 |
Entry.Ty = Type::getInt8Ty(*getContext()); |
0 |
7980 |
Entry.Ty = Type::getInt8Ty(*getContext()); |
0 |
| 7981 |
Entry.Node = Value; |
0 |
7981 |
Entry.Node = Value; |
0 |
| 7982 |
Args.push_back(Entry); |
0 |
7982 |
Args.push_back(Entry); |
0 |
| 7983 |
|
--- |
7983 |
|
--- |
| 7984 |
Entry.Ty = SizeTy; |
0 |
7984 |
Entry.Ty = SizeTy; |
0 |
| 7985 |
Entry.Node = Size; |
0 |
7985 |
Entry.Node = Size; |
0 |
| 7986 |
Args.push_back(Entry); |
0 |
7986 |
Args.push_back(Entry); |
0 |
| 7987 |
|
--- |
7987 |
|
--- |
| 7988 |
RTLIB::Libcall LibraryCall = |
--- |
7988 |
RTLIB::Libcall LibraryCall = |
--- |
| 7989 |
RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElemSz); |
0 |
7989 |
RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElemSz); |
0 |
| 7990 |
if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) |
0 |
7990 |
if (LibraryCall == RTLIB::UNKNOWN_LIBCALL) |
0 |
| 7991 |
report_fatal_error("Unsupported element size"); |
0 |
7991 |
report_fatal_error("Unsupported element size"); |
0 |
| 7992 |
|
--- |
7992 |
|
--- |
| 7993 |
TargetLowering::CallLoweringInfo CLI(*this); |
0 |
7993 |
TargetLowering::CallLoweringInfo CLI(*this); |
0 |
| 7994 |
CLI.setDebugLoc(dl) |
0 |
7994 |
CLI.setDebugLoc(dl) |
0 |
| 7995 |
.setChain(Chain) |
0 |
7995 |
.setChain(Chain) |
0 |
| 7996 |
.setLibCallee(TLI->getLibcallCallingConv(LibraryCall), |
0 |
7996 |
.setLibCallee(TLI->getLibcallCallingConv(LibraryCall), |
0 |
| 7997 |
Type::getVoidTy(*getContext()), |
0 |
7997 |
Type::getVoidTy(*getContext()), |
0 |
| 7998 |
getExternalSymbol(TLI->getLibcallName(LibraryCall), |
0 |
7998 |
getExternalSymbol(TLI->getLibcallName(LibraryCall), |
0 |
| 7999 |
TLI->getPointerTy(getDataLayout())), |
0 |
7999 |
TLI->getPointerTy(getDataLayout())), |
0 |
| 8000 |
std::move(Args)) |
0 |
8000 |
std::move(Args)) |
0 |
| 8001 |
.setDiscardResult() |
0 |
8001 |
.setDiscardResult() |
0 |
| 8002 |
.setTailCall(isTailCall); |
0 |
8002 |
.setTailCall(isTailCall); |
0 |
| 8003 |
|
--- |
8003 |
|
--- |
| 8004 |
std::pair CallResult = TLI->LowerCallTo(CLI); |
0 |
8004 |
std::pair CallResult = TLI->LowerCallTo(CLI); |
0 |
| 8005 |
return CallResult.second; |
0 |
8005 |
return CallResult.second; |
0 |
| 8006 |
} |
0 |
8006 |
} |
0 |
| 8007 |
|
--- |
8007 |
|
--- |
| 8008 |
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, |
0 |
8008 |
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, |
0 |
| 8009 |
SDVTList VTList, ArrayRef Ops, |
--- |
8009 |
SDVTList VTList, ArrayRef Ops, |
--- |
| 8010 |
MachineMemOperand *MMO) { |
--- |
8010 |
MachineMemOperand *MMO) { |
--- |
| 8011 |
FoldingSetNodeID ID; |
0 |
8011 |
FoldingSetNodeID ID; |
0 |
| 8012 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
8012 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
| 8013 |
AddNodeIDNode(ID, Opcode, VTList, Ops); |
0 |
8013 |
AddNodeIDNode(ID, Opcode, VTList, Ops); |
0 |
| 8014 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
8014 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
| 8015 |
ID.AddInteger(MMO->getFlags()); |
0 |
8015 |
ID.AddInteger(MMO->getFlags()); |
0 |
| 8016 |
void* IP = nullptr; |
0 |
8016 |
void* IP = nullptr; |
0 |
| 8017 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
8017 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
| 8018 |
cast(E)->refineAlignment(MMO); |
0 |
8018 |
cast(E)->refineAlignment(MMO); |
0 |
| 8019 |
return SDValue(E, 0); |
0 |
8019 |
return SDValue(E, 0); |
0 |
| 8020 |
} |
--- |
8020 |
} |
--- |
| 8021 |
|
--- |
8021 |
|
--- |
| 8022 |
auto *N = newSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), |
0 |
8022 |
auto *N = newSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), |
0 |
| 8023 |
VTList, MemVT, MMO); |
--- |
8023 |
VTList, MemVT, MMO); |
--- |
| 8024 |
createOperands(N, Ops); |
0 |
8024 |
createOperands(N, Ops); |
0 |
| 8025 |
|
--- |
8025 |
|
--- |
| 8026 |
CSEMap.InsertNode(N, IP); |
0 |
8026 |
CSEMap.InsertNode(N, IP); |
0 |
| 8027 |
InsertNode(N); |
0 |
8027 |
InsertNode(N); |
0 |
| 8028 |
return SDValue(N, 0); |
0 |
8028 |
return SDValue(N, 0); |
0 |
| 8029 |
} |
0 |
8029 |
} |
0 |
| 8030 |
|
--- |
8030 |
|
--- |
| 8031 |
SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, |
0 |
8031 |
SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, |
0 |
| 8032 |
EVT MemVT, SDVTList VTs, SDValue Chain, |
--- |
8032 |
EVT MemVT, SDVTList VTs, SDValue Chain, |
--- |
| 8033 |
SDValue Ptr, SDValue Cmp, SDValue Swp, |
--- |
8033 |
SDValue Ptr, SDValue Cmp, SDValue Swp, |
--- |
| 8034 |
MachineMemOperand *MMO) { |
--- |
8034 |
MachineMemOperand *MMO) { |
--- |
| 8035 |
assert(Opcode == ISD::ATOMIC_CMP_SWAP || |
0 |
8035 |
assert(Opcode == ISD::ATOMIC_CMP_SWAP || |
0 |
| 8036 |
Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); |
--- |
8036 |
Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); |
--- |
| 8037 |
assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); |
0 |
8037 |
assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); |
0 |
| 8038 |
|
--- |
8038 |
|
--- |
| 8039 |
SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; |
0 |
8039 |
SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; |
0 |
| 8040 |
return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); |
0 |
8040 |
return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); |
0 |
| 8041 |
} |
--- |
8041 |
} |
--- |
| 8042 |
|
--- |
8042 |
|
--- |
| 8043 |
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, |
0 |
8043 |
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, |
0 |
| 8044 |
SDValue Chain, SDValue Ptr, SDValue Val, |
--- |
8044 |
SDValue Chain, SDValue Ptr, SDValue Val, |
--- |
| 8045 |
MachineMemOperand *MMO) { |
--- |
8045 |
MachineMemOperand *MMO) { |
--- |
| 8046 |
assert((Opcode == ISD::ATOMIC_LOAD_ADD || |
0 |
8046 |
assert((Opcode == ISD::ATOMIC_LOAD_ADD || |
0 |
| 8047 |
Opcode == ISD::ATOMIC_LOAD_SUB || |
--- |
8047 |
Opcode == ISD::ATOMIC_LOAD_SUB || |
--- |
| 8048 |
Opcode == ISD::ATOMIC_LOAD_AND || |
--- |
8048 |
Opcode == ISD::ATOMIC_LOAD_AND || |
--- |
| 8049 |
Opcode == ISD::ATOMIC_LOAD_CLR || |
--- |
8049 |
Opcode == ISD::ATOMIC_LOAD_CLR || |
--- |
| 8050 |
Opcode == ISD::ATOMIC_LOAD_OR || |
--- |
8050 |
Opcode == ISD::ATOMIC_LOAD_OR || |
--- |
| 8051 |
Opcode == ISD::ATOMIC_LOAD_XOR || |
--- |
8051 |
Opcode == ISD::ATOMIC_LOAD_XOR || |
--- |
| 8052 |
Opcode == ISD::ATOMIC_LOAD_NAND || |
--- |
8052 |
Opcode == ISD::ATOMIC_LOAD_NAND || |
--- |
| 8053 |
Opcode == ISD::ATOMIC_LOAD_MIN || |
--- |
8053 |
Opcode == ISD::ATOMIC_LOAD_MIN || |
--- |
| 8054 |
Opcode == ISD::ATOMIC_LOAD_MAX || |
--- |
8054 |
Opcode == ISD::ATOMIC_LOAD_MAX || |
--- |
| 8055 |
Opcode == ISD::ATOMIC_LOAD_UMIN || |
--- |
8055 |
Opcode == ISD::ATOMIC_LOAD_UMIN || |
--- |
| 8056 |
Opcode == ISD::ATOMIC_LOAD_UMAX || |
--- |
8056 |
Opcode == ISD::ATOMIC_LOAD_UMAX || |
--- |
| 8057 |
Opcode == ISD::ATOMIC_LOAD_FADD || |
--- |
8057 |
Opcode == ISD::ATOMIC_LOAD_FADD || |
--- |
| 8058 |
Opcode == ISD::ATOMIC_LOAD_FSUB || |
--- |
8058 |
Opcode == ISD::ATOMIC_LOAD_FSUB || |
--- |
| 8059 |
Opcode == ISD::ATOMIC_LOAD_FMAX || |
--- |
8059 |
Opcode == ISD::ATOMIC_LOAD_FMAX || |
--- |
| 8060 |
Opcode == ISD::ATOMIC_LOAD_FMIN || |
--- |
8060 |
Opcode == ISD::ATOMIC_LOAD_FMIN || |
--- |
| 8061 |
Opcode == ISD::ATOMIC_LOAD_UINC_WRAP || |
--- |
8061 |
Opcode == ISD::ATOMIC_LOAD_UINC_WRAP || |
--- |
| 8062 |
Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP || |
--- |
8062 |
Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP || |
--- |
| 8063 |
Opcode == ISD::ATOMIC_SWAP || |
--- |
8063 |
Opcode == ISD::ATOMIC_SWAP || |
--- |
| 8064 |
Opcode == ISD::ATOMIC_STORE) && |
--- |
8064 |
Opcode == ISD::ATOMIC_STORE) && |
--- |
| 8065 |
"Invalid Atomic Op"); |
--- |
8065 |
"Invalid Atomic Op"); |
--- |
| 8066 |
|
--- |
8066 |
|
--- |
| 8067 |
EVT VT = Val.getValueType(); |
0 |
8067 |
EVT VT = Val.getValueType(); |
0 |
| 8068 |
|
--- |
8068 |
|
--- |
| 8069 |
SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : |
0 |
8069 |
SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : |
0 |
| 8070 |
getVTList(VT, MVT::Other); |
0 |
8070 |
getVTList(VT, MVT::Other); |
0 |
| 8071 |
SDValue Ops[] = {Chain, Ptr, Val}; |
0 |
8071 |
SDValue Ops[] = {Chain, Ptr, Val}; |
0 |
| 8072 |
return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); |
0 |
8072 |
return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); |
0 |
| 8073 |
} |
--- |
8073 |
} |
--- |
| 8074 |
|
--- |
8074 |
|
--- |
| 8075 |
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, |
0 |
8075 |
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, |
0 |
| 8076 |
EVT VT, SDValue Chain, SDValue Ptr, |
--- |
8076 |
EVT VT, SDValue Chain, SDValue Ptr, |
--- |
| 8077 |
MachineMemOperand *MMO) { |
--- |
8077 |
MachineMemOperand *MMO) { |
--- |
| 8078 |
assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); |
0 |
8078 |
assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); |
0 |
| 8079 |
|
--- |
8079 |
|
--- |
| 8080 |
SDVTList VTs = getVTList(VT, MVT::Other); |
0 |
8080 |
SDVTList VTs = getVTList(VT, MVT::Other); |
0 |
| 8081 |
SDValue Ops[] = {Chain, Ptr}; |
0 |
8081 |
SDValue Ops[] = {Chain, Ptr}; |
0 |
| 8082 |
return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); |
0 |
8082 |
return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); |
0 |
| 8083 |
} |
--- |
8083 |
} |
--- |
| 8084 |
|
--- |
8084 |
|
--- |
| 8085 |
/// getMergeValues - Create a MERGE_VALUES node from the given operands. |
--- |
8085 |
/// getMergeValues - Create a MERGE_VALUES node from the given operands. |
--- |
| 8086 |
SDValue SelectionDAG::getMergeValues(ArrayRef Ops, const SDLoc &dl) { |
2 |
8086 |
SDValue SelectionDAG::getMergeValues(ArrayRef Ops, const SDLoc &dl) { |
2 |
| 8087 |
if (Ops.size() == 1) |
2 |
8087 |
if (Ops.size() == 1) |
2 |
| 8088 |
return Ops[0]; |
2 |
8088 |
return Ops[0]; |
2 |
| 8089 |
|
--- |
8089 |
|
--- |
| 8090 |
SmallVector VTs; |
0 |
8090 |
SmallVector VTs; |
0 |
| 8091 |
VTs.reserve(Ops.size()); |
0 |
8091 |
VTs.reserve(Ops.size()); |
0 |
| 8092 |
for (const SDValue &Op : Ops) |
0 |
8092 |
for (const SDValue &Op : Ops) |
0 |
| 8093 |
VTs.push_back(Op.getValueType()); |
0 |
8093 |
VTs.push_back(Op.getValueType()); |
0 |
| 8094 |
return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops); |
0 |
8094 |
return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops); |
0 |
| 8095 |
} |
0 |
8095 |
} |
0 |
| 8096 |
|
--- |
8096 |
|
--- |
| 8097 |
SDValue SelectionDAG::getMemIntrinsicNode( |
0 |
8097 |
SDValue SelectionDAG::getMemIntrinsicNode( |
0 |
| 8098 |
unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef Ops, |
--- |
8098 |
unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef Ops, |
--- |
| 8099 |
EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, |
--- |
8099 |
EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, |
--- |
| 8100 |
MachineMemOperand::Flags Flags, uint64_t Size, const AAMDNodes &AAInfo) { |
--- |
8100 |
MachineMemOperand::Flags Flags, uint64_t Size, const AAMDNodes &AAInfo) { |
--- |
| 8101 |
if (!Size && MemVT.isScalableVector()) |
0 |
8101 |
if (!Size && MemVT.isScalableVector()) |
0 |
| 8102 |
Size = MemoryLocation::UnknownSize; |
0 |
8102 |
Size = MemoryLocation::UnknownSize; |
0 |
| 8103 |
else if (!Size) |
0 |
8103 |
else if (!Size) |
0 |
| 8104 |
Size = MemVT.getStoreSize(); |
0 |
8104 |
Size = MemVT.getStoreSize(); |
0 |
| 8105 |
|
--- |
8105 |
|
--- |
| 8106 |
MachineFunction &MF = getMachineFunction(); |
0 |
8106 |
MachineFunction &MF = getMachineFunction(); |
0 |
| 8107 |
MachineMemOperand *MMO = |
--- |
8107 |
MachineMemOperand *MMO = |
--- |
| 8108 |
MF.getMachineMemOperand(PtrInfo, Flags, Size, Alignment, AAInfo); |
0 |
8108 |
MF.getMachineMemOperand(PtrInfo, Flags, Size, Alignment, AAInfo); |
0 |
| 8109 |
|
--- |
8109 |
|
--- |
| 8110 |
return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO); |
0 |
8110 |
return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO); |
0 |
| 8111 |
} |
--- |
8111 |
} |
--- |
| 8112 |
|
--- |
8112 |
|
--- |
| 8113 |
SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, |
0 |
8113 |
SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, |
0 |
| 8114 |
SDVTList VTList, |
--- |
8114 |
SDVTList VTList, |
--- |
| 8115 |
ArrayRef Ops, EVT MemVT, |
--- |
8115 |
ArrayRef Ops, EVT MemVT, |
--- |
| 8116 |
MachineMemOperand *MMO) { |
--- |
8116 |
MachineMemOperand *MMO) { |
--- |
| 8117 |
assert((Opcode == ISD::INTRINSIC_VOID || |
0 |
8117 |
assert((Opcode == ISD::INTRINSIC_VOID || |
0 |
| 8118 |
Opcode == ISD::INTRINSIC_W_CHAIN || |
--- |
8118 |
Opcode == ISD::INTRINSIC_W_CHAIN || |
--- |
| 8119 |
Opcode == ISD::PREFETCH || |
--- |
8119 |
Opcode == ISD::PREFETCH || |
--- |
| 8120 |
(Opcode <= (unsigned)std::numeric_limits::max() && |
--- |
8120 |
(Opcode <= (unsigned)std::numeric_limits::max() && |
--- |
| 8121 |
(int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && |
--- |
8121 |
(int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && |
--- |
| 8122 |
"Opcode is not a memory-accessing opcode!"); |
--- |
8122 |
"Opcode is not a memory-accessing opcode!"); |
--- |
| 8123 |
|
--- |
8123 |
|
--- |
| 8124 |
// Memoize the node unless it returns a flag. |
--- |
8124 |
// Memoize the node unless it returns a flag. |
--- |
| 8125 |
MemIntrinsicSDNode *N; |
--- |
8125 |
MemIntrinsicSDNode *N; |
--- |
| 8126 |
if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { |
0 |
8126 |
if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { |
0 |
| 8127 |
FoldingSetNodeID ID; |
0 |
8127 |
FoldingSetNodeID ID; |
0 |
| 8128 |
AddNodeIDNode(ID, Opcode, VTList, Ops); |
0 |
8128 |
AddNodeIDNode(ID, Opcode, VTList, Ops); |
0 |
| 8129 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
8129 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
| 8130 |
Opcode, dl.getIROrder(), VTList, MemVT, MMO)); |
--- |
8130 |
Opcode, dl.getIROrder(), VTList, MemVT, MMO)); |
--- |
| 8131 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
8131 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
| 8132 |
ID.AddInteger(MMO->getFlags()); |
0 |
8132 |
ID.AddInteger(MMO->getFlags()); |
0 |
| 8133 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
8133 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
| 8134 |
void *IP = nullptr; |
0 |
8134 |
void *IP = nullptr; |
0 |
| 8135 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
8135 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
| 8136 |
cast(E)->refineAlignment(MMO); |
0 |
8136 |
cast(E)->refineAlignment(MMO); |
0 |
| 8137 |
return SDValue(E, 0); |
0 |
8137 |
return SDValue(E, 0); |
0 |
| 8138 |
} |
--- |
8138 |
} |
--- |
| 8139 |
|
--- |
8139 |
|
--- |
| 8140 |
N = newSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), |
0 |
8140 |
N = newSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), |
0 |
| 8141 |
VTList, MemVT, MMO); |
--- |
8141 |
VTList, MemVT, MMO); |
--- |
| 8142 |
createOperands(N, Ops); |
0 |
8142 |
createOperands(N, Ops); |
0 |
| 8143 |
|
--- |
8143 |
|
--- |
| 8144 |
CSEMap.InsertNode(N, IP); |
0 |
8144 |
CSEMap.InsertNode(N, IP); |
0 |
| 8145 |
} else { |
0 |
8145 |
} else { |
0 |
| 8146 |
N = newSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), |
0 |
8146 |
N = newSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), |
0 |
| 8147 |
VTList, MemVT, MMO); |
--- |
8147 |
VTList, MemVT, MMO); |
--- |
| 8148 |
createOperands(N, Ops); |
0 |
8148 |
createOperands(N, Ops); |
0 |
| 8149 |
} |
--- |
8149 |
} |
--- |
| 8150 |
InsertNode(N); |
0 |
8150 |
InsertNode(N); |
0 |
| 8151 |
SDValue V(N, 0); |
0 |
8151 |
SDValue V(N, 0); |
0 |
| 8152 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
8152 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 8153 |
return V; |
0 |
8153 |
return V; |
0 |
| 8154 |
} |
--- |
8154 |
} |
--- |
| 8155 |
|
--- |
8155 |
|
--- |
| 8156 |
SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl, |
0 |
8156 |
SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl, |
0 |
| 8157 |
SDValue Chain, int FrameIndex, |
--- |
8157 |
SDValue Chain, int FrameIndex, |
--- |
| 8158 |
int64_t Size, int64_t Offset) { |
--- |
8158 |
int64_t Size, int64_t Offset) { |
--- |
| 8159 |
const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END; |
0 |
8159 |
const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END; |
0 |
| 8160 |
const auto VTs = getVTList(MVT::Other); |
0 |
8160 |
const auto VTs = getVTList(MVT::Other); |
0 |
| 8161 |
SDValue Ops[2] = { |
--- |
8161 |
SDValue Ops[2] = { |
--- |
| 8162 |
Chain, |
--- |
8162 |
Chain, |
--- |
| 8163 |
getFrameIndex(FrameIndex, |
0 |
8163 |
getFrameIndex(FrameIndex, |
0 |
| 8164 |
getTargetLoweringInfo().getFrameIndexTy(getDataLayout()), |
0 |
8164 |
getTargetLoweringInfo().getFrameIndexTy(getDataLayout()), |
0 |
| 8165 |
true)}; |
0 |
8165 |
true)}; |
0 |
| 8166 |
|
--- |
8166 |
|
--- |
| 8167 |
FoldingSetNodeID ID; |
0 |
8167 |
FoldingSetNodeID ID; |
0 |
| 8168 |
AddNodeIDNode(ID, Opcode, VTs, Ops); |
0 |
8168 |
AddNodeIDNode(ID, Opcode, VTs, Ops); |
0 |
| 8169 |
ID.AddInteger(FrameIndex); |
0 |
8169 |
ID.AddInteger(FrameIndex); |
0 |
| 8170 |
ID.AddInteger(Size); |
0 |
8170 |
ID.AddInteger(Size); |
0 |
| 8171 |
ID.AddInteger(Offset); |
0 |
8171 |
ID.AddInteger(Offset); |
0 |
| 8172 |
void *IP = nullptr; |
0 |
8172 |
void *IP = nullptr; |
0 |
| 8173 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) |
0 |
8173 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) |
0 |
| 8174 |
return SDValue(E, 0); |
0 |
8174 |
return SDValue(E, 0); |
0 |
| 8175 |
|
--- |
8175 |
|
--- |
| 8176 |
LifetimeSDNode *N = newSDNode( |
0 |
8176 |
LifetimeSDNode *N = newSDNode( |
0 |
| 8177 |
Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset); |
0 |
8177 |
Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset); |
0 |
| 8178 |
createOperands(N, Ops); |
0 |
8178 |
createOperands(N, Ops); |
0 |
| 8179 |
CSEMap.InsertNode(N, IP); |
0 |
8179 |
CSEMap.InsertNode(N, IP); |
0 |
| 8180 |
InsertNode(N); |
0 |
8180 |
InsertNode(N); |
0 |
| 8181 |
SDValue V(N, 0); |
0 |
8181 |
SDValue V(N, 0); |
0 |
| 8182 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
8182 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 8183 |
return V; |
0 |
8183 |
return V; |
0 |
| 8184 |
} |
0 |
8184 |
} |
0 |
| 8185 |
|
--- |
8185 |
|
--- |
| 8186 |
SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, |
0 |
8186 |
SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, |
0 |
| 8187 |
uint64_t Guid, uint64_t Index, |
--- |
8187 |
uint64_t Guid, uint64_t Index, |
--- |
| 8188 |
uint32_t Attr) { |
--- |
8188 |
uint32_t Attr) { |
--- |
| 8189 |
const unsigned Opcode = ISD::PSEUDO_PROBE; |
0 |
8189 |
const unsigned Opcode = ISD::PSEUDO_PROBE; |
0 |
| 8190 |
const auto VTs = getVTList(MVT::Other); |
0 |
8190 |
const auto VTs = getVTList(MVT::Other); |
0 |
| 8191 |
SDValue Ops[] = {Chain}; |
0 |
8191 |
SDValue Ops[] = {Chain}; |
0 |
| 8192 |
FoldingSetNodeID ID; |
0 |
8192 |
FoldingSetNodeID ID; |
0 |
| 8193 |
AddNodeIDNode(ID, Opcode, VTs, Ops); |
0 |
8193 |
AddNodeIDNode(ID, Opcode, VTs, Ops); |
0 |
| 8194 |
ID.AddInteger(Guid); |
0 |
8194 |
ID.AddInteger(Guid); |
0 |
| 8195 |
ID.AddInteger(Index); |
0 |
8195 |
ID.AddInteger(Index); |
0 |
| 8196 |
void *IP = nullptr; |
0 |
8196 |
void *IP = nullptr; |
0 |
| 8197 |
if (SDNode *E = FindNodeOrInsertPos(ID, Dl, IP)) |
0 |
8197 |
if (SDNode *E = FindNodeOrInsertPos(ID, Dl, IP)) |
0 |
| 8198 |
return SDValue(E, 0); |
0 |
8198 |
return SDValue(E, 0); |
0 |
| 8199 |
|
--- |
8199 |
|
--- |
| 8200 |
auto *N = newSDNode( |
0 |
8200 |
auto *N = newSDNode( |
0 |
| 8201 |
Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr); |
0 |
8201 |
Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr); |
0 |
| 8202 |
createOperands(N, Ops); |
0 |
8202 |
createOperands(N, Ops); |
0 |
| 8203 |
CSEMap.InsertNode(N, IP); |
0 |
8203 |
CSEMap.InsertNode(N, IP); |
0 |
| 8204 |
InsertNode(N); |
0 |
8204 |
InsertNode(N); |
0 |
| 8205 |
SDValue V(N, 0); |
0 |
8205 |
SDValue V(N, 0); |
0 |
| 8206 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
8206 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 8207 |
return V; |
0 |
8207 |
return V; |
0 |
| 8208 |
} |
0 |
8208 |
} |
0 |
| 8209 |
|
--- |
8209 |
|
--- |
| 8210 |
/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a |
--- |
8210 |
/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a |
--- |
| 8211 |
/// MachinePointerInfo record from it. This is particularly useful because the |
--- |
8211 |
/// MachinePointerInfo record from it. This is particularly useful because the |
--- |
| 8212 |
/// code generator has many cases where it doesn't bother passing in a |
--- |
8212 |
/// code generator has many cases where it doesn't bother passing in a |
--- |
| 8213 |
/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". |
--- |
8213 |
/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". |
--- |
| 8214 |
static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, |
0 |
8214 |
static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, |
0 |
| 8215 |
SelectionDAG &DAG, SDValue Ptr, |
--- |
8215 |
SelectionDAG &DAG, SDValue Ptr, |
--- |
| 8216 |
int64_t Offset = 0) { |
--- |
8216 |
int64_t Offset = 0) { |
--- |
| 8217 |
// If this is FI+Offset, we can model it. |
--- |
8217 |
// If this is FI+Offset, we can model it. |
--- |
| 8218 |
if (const FrameIndexSDNode *FI = dyn_cast(Ptr)) |
0 |
8218 |
if (const FrameIndexSDNode *FI = dyn_cast(Ptr)) |
0 |
| 8219 |
return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), |
0 |
8219 |
return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), |
0 |
| 8220 |
FI->getIndex(), Offset); |
0 |
8220 |
FI->getIndex(), Offset); |
0 |
| 8221 |
|
--- |
8221 |
|
--- |
| 8222 |
// If this is (FI+Offset1)+Offset2, we can model it. |
--- |
8222 |
// If this is (FI+Offset1)+Offset2, we can model it. |
--- |
| 8223 |
if (Ptr.getOpcode() != ISD::ADD || |
0 |
8223 |
if (Ptr.getOpcode() != ISD::ADD || |
0 |
| 8224 |
!isa(Ptr.getOperand(1)) || |
0 |
8224 |
!isa(Ptr.getOperand(1)) || |
0 |
| 8225 |
!isa(Ptr.getOperand(0))) |
0 |
8225 |
!isa(Ptr.getOperand(0))) |
0 |
| 8226 |
return Info; |
0 |
8226 |
return Info; |
0 |
| 8227 |
|
--- |
8227 |
|
--- |
| 8228 |
int FI = cast(Ptr.getOperand(0))->getIndex(); |
0 |
8228 |
int FI = cast(Ptr.getOperand(0))->getIndex(); |
0 |
| 8229 |
return MachinePointerInfo::getFixedStack( |
0 |
8229 |
return MachinePointerInfo::getFixedStack( |
0 |
| 8230 |
DAG.getMachineFunction(), FI, |
--- |
8230 |
DAG.getMachineFunction(), FI, |
--- |
| 8231 |
Offset + cast(Ptr.getOperand(1))->getSExtValue()); |
0 |
8231 |
Offset + cast(Ptr.getOperand(1))->getSExtValue()); |
0 |
| 8232 |
} |
--- |
8232 |
} |
--- |
| 8233 |
|
--- |
8233 |
|
--- |
| 8234 |
/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a |
--- |
8234 |
/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a |
--- |
| 8235 |
/// MachinePointerInfo record from it. This is particularly useful because the |
--- |
8235 |
/// MachinePointerInfo record from it. This is particularly useful because the |
--- |
| 8236 |
/// code generator has many cases where it doesn't bother passing in a |
--- |
8236 |
/// code generator has many cases where it doesn't bother passing in a |
--- |
| 8237 |
/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". |
--- |
8237 |
/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". |
--- |
| 8238 |
static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, |
0 |
8238 |
static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, |
0 |
| 8239 |
SelectionDAG &DAG, SDValue Ptr, |
--- |
8239 |
SelectionDAG &DAG, SDValue Ptr, |
--- |
| 8240 |
SDValue OffsetOp) { |
--- |
8240 |
SDValue OffsetOp) { |
--- |
| 8241 |
// If the 'Offset' value isn't a constant, we can't handle this. |
--- |
8241 |
// If the 'Offset' value isn't a constant, we can't handle this. |
--- |
| 8242 |
if (ConstantSDNode *OffsetNode = dyn_cast(OffsetOp)) |
0 |
8242 |
if (ConstantSDNode *OffsetNode = dyn_cast(OffsetOp)) |
0 |
| 8243 |
return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue()); |
0 |
8243 |
return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue()); |
0 |
| 8244 |
if (OffsetOp.isUndef()) |
0 |
8244 |
if (OffsetOp.isUndef()) |
0 |
| 8245 |
return InferPointerInfo(Info, DAG, Ptr); |
0 |
8245 |
return InferPointerInfo(Info, DAG, Ptr); |
0 |
| 8246 |
return Info; |
0 |
8246 |
return Info; |
0 |
| 8247 |
} |
--- |
8247 |
} |
--- |
| 8248 |
|
--- |
8248 |
|
--- |
| 8249 |
SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, |
5 |
8249 |
SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, |
5 |
| 8250 |
EVT VT, const SDLoc &dl, SDValue Chain, |
--- |
8250 |
EVT VT, const SDLoc &dl, SDValue Chain, |
--- |
| 8251 |
SDValue Ptr, SDValue Offset, |
--- |
8251 |
SDValue Ptr, SDValue Offset, |
--- |
| 8252 |
MachinePointerInfo PtrInfo, EVT MemVT, |
--- |
8252 |
MachinePointerInfo PtrInfo, EVT MemVT, |
--- |
| 8253 |
Align Alignment, |
--- |
8253 |
Align Alignment, |
--- |
| 8254 |
MachineMemOperand::Flags MMOFlags, |
--- |
8254 |
MachineMemOperand::Flags MMOFlags, |
--- |
| 8255 |
const AAMDNodes &AAInfo, const MDNode *Ranges) { |
--- |
8255 |
const AAMDNodes &AAInfo, const MDNode *Ranges) { |
--- |
| 8256 |
assert(Chain.getValueType() == MVT::Other && |
5 |
8256 |
assert(Chain.getValueType() == MVT::Other && |
5 |
| 8257 |
"Invalid chain type"); |
--- |
8257 |
"Invalid chain type"); |
--- |
| 8258 |
|
--- |
8258 |
|
--- |
| 8259 |
MMOFlags |= MachineMemOperand::MOLoad; |
5 |
8259 |
MMOFlags |= MachineMemOperand::MOLoad; |
5 |
| 8260 |
assert((MMOFlags & MachineMemOperand::MOStore) == 0); |
5 |
8260 |
assert((MMOFlags & MachineMemOperand::MOStore) == 0); |
5 |
| 8261 |
// If we don't have a PtrInfo, infer the trivial frame index case to simplify |
--- |
8261 |
// If we don't have a PtrInfo, infer the trivial frame index case to simplify |
--- |
| 8262 |
// clients. |
--- |
8262 |
// clients. |
--- |
| 8263 |
if (PtrInfo.V.isNull()) |
5 |
8263 |
if (PtrInfo.V.isNull()) |
5 |
| 8264 |
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset); |
0 |
8264 |
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset); |
0 |
| 8265 |
|
--- |
8265 |
|
--- |
| 8266 |
uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize()); |
5 |
8266 |
uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize()); |
5 |
| 8267 |
MachineFunction &MF = getMachineFunction(); |
5 |
8267 |
MachineFunction &MF = getMachineFunction(); |
5 |
| 8268 |
MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, |
5 |
8268 |
MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, |
5 |
| 8269 |
Alignment, AAInfo, Ranges); |
--- |
8269 |
Alignment, AAInfo, Ranges); |
--- |
| 8270 |
return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); |
5 |
8270 |
return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); |
5 |
| 8271 |
} |
--- |
8271 |
} |
--- |
| 8272 |
|
--- |
8272 |
|
--- |
| 8273 |
SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, |
5 |
8273 |
SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, |
5 |
| 8274 |
EVT VT, const SDLoc &dl, SDValue Chain, |
--- |
8274 |
EVT VT, const SDLoc &dl, SDValue Chain, |
--- |
| 8275 |
SDValue Ptr, SDValue Offset, EVT MemVT, |
--- |
8275 |
SDValue Ptr, SDValue Offset, EVT MemVT, |
--- |
| 8276 |
MachineMemOperand *MMO) { |
--- |
8276 |
MachineMemOperand *MMO) { |
--- |
| 8277 |
if (VT == MemVT) { |
5 |
8277 |
if (VT == MemVT) { |
5 |
| 8278 |
ExtType = ISD::NON_EXTLOAD; |
5 |
8278 |
ExtType = ISD::NON_EXTLOAD; |
5 |
| 8279 |
} else if (ExtType == ISD::NON_EXTLOAD) { |
0 |
8279 |
} else if (ExtType == ISD::NON_EXTLOAD) { |
0 |
| 8280 |
assert(VT == MemVT && "Non-extending load from different memory type!"); |
0 |
8280 |
assert(VT == MemVT && "Non-extending load from different memory type!"); |
0 |
| 8281 |
} else { |
--- |
8281 |
} else { |
--- |
| 8282 |
// Extending load. |
--- |
8282 |
// Extending load. |
--- |
| 8283 |
assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && |
0 |
8283 |
assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && |
0 |
| 8284 |
"Should only be an extending load, not truncating!"); |
--- |
8284 |
"Should only be an extending load, not truncating!"); |
--- |
| 8285 |
assert(VT.isInteger() == MemVT.isInteger() && |
0 |
8285 |
assert(VT.isInteger() == MemVT.isInteger() && |
0 |
| 8286 |
"Cannot convert from FP to Int or Int -> FP!"); |
--- |
8286 |
"Cannot convert from FP to Int or Int -> FP!"); |
--- |
| 8287 |
assert(VT.isVector() == MemVT.isVector() && |
0 |
8287 |
assert(VT.isVector() == MemVT.isVector() && |
0 |
| 8288 |
"Cannot use an ext load to convert to or from a vector!"); |
--- |
8288 |
"Cannot use an ext load to convert to or from a vector!"); |
--- |
| 8289 |
assert((!VT.isVector() || |
0 |
8289 |
assert((!VT.isVector() || |
0 |
| 8290 |
VT.getVectorElementCount() == MemVT.getVectorElementCount()) && |
--- |
8290 |
VT.getVectorElementCount() == MemVT.getVectorElementCount()) && |
--- |
| 8291 |
"Cannot use an ext load to change the number of vector elements!"); |
--- |
8291 |
"Cannot use an ext load to change the number of vector elements!"); |
--- |
| 8292 |
} |
--- |
8292 |
} |
--- |
| 8293 |
|
--- |
8293 |
|
--- |
| 8294 |
bool Indexed = AM != ISD::UNINDEXED; |
5 |
8294 |
bool Indexed = AM != ISD::UNINDEXED; |
5 |
| 8295 |
assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!"); |
5 |
8295 |
assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!"); |
5 |
| 8296 |
|
--- |
8296 |
|
--- |
| 8297 |
SDVTList VTs = Indexed ? |
--- |
8297 |
SDVTList VTs = Indexed ? |
--- |
| 8298 |
getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other); |
5 |
8298 |
getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other); |
5 |
| 8299 |
SDValue Ops[] = { Chain, Ptr, Offset }; |
5 |
8299 |
SDValue Ops[] = { Chain, Ptr, Offset }; |
5 |
| 8300 |
FoldingSetNodeID ID; |
5 |
8300 |
FoldingSetNodeID ID; |
5 |
| 8301 |
AddNodeIDNode(ID, ISD::LOAD, VTs, Ops); |
5 |
8301 |
AddNodeIDNode(ID, ISD::LOAD, VTs, Ops); |
5 |
| 8302 |
ID.AddInteger(MemVT.getRawBits()); |
5 |
8302 |
ID.AddInteger(MemVT.getRawBits()); |
5 |
| 8303 |
ID.AddInteger(getSyntheticNodeSubclassData( |
5 |
8303 |
ID.AddInteger(getSyntheticNodeSubclassData( |
5 |
| 8304 |
dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO)); |
--- |
8304 |
dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO)); |
--- |
| 8305 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
5 |
8305 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
5 |
| 8306 |
ID.AddInteger(MMO->getFlags()); |
5 |
8306 |
ID.AddInteger(MMO->getFlags()); |
5 |
| 8307 |
void *IP = nullptr; |
5 |
8307 |
void *IP = nullptr; |
5 |
| 8308 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
5 |
8308 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
5 |
| 8309 |
cast(E)->refineAlignment(MMO); |
0 |
8309 |
cast(E)->refineAlignment(MMO); |
0 |
| 8310 |
return SDValue(E, 0); |
0 |
8310 |
return SDValue(E, 0); |
0 |
| 8311 |
} |
--- |
8311 |
} |
--- |
| 8312 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, |
5 |
8312 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, |
5 |
| 8313 |
ExtType, MemVT, MMO); |
--- |
8313 |
ExtType, MemVT, MMO); |
--- |
| 8314 |
createOperands(N, Ops); |
5 |
8314 |
createOperands(N, Ops); |
5 |
| 8315 |
|
--- |
8315 |
|
--- |
| 8316 |
CSEMap.InsertNode(N, IP); |
5 |
8316 |
CSEMap.InsertNode(N, IP); |
5 |
| 8317 |
InsertNode(N); |
5 |
8317 |
InsertNode(N); |
5 |
| 8318 |
SDValue V(N, 0); |
5 |
8318 |
SDValue V(N, 0); |
5 |
| 8319 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
5 |
8319 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
5 |
| 8320 |
return V; |
5 |
8320 |
return V; |
5 |
| 8321 |
} |
5 |
8321 |
} |
5 |
| 8322 |
|
--- |
8322 |
|
--- |
| 8323 |
SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain, |
5 |
8323 |
SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain, |
5 |
| 8324 |
SDValue Ptr, MachinePointerInfo PtrInfo, |
--- |
8324 |
SDValue Ptr, MachinePointerInfo PtrInfo, |
--- |
| 8325 |
MaybeAlign Alignment, |
--- |
8325 |
MaybeAlign Alignment, |
--- |
| 8326 |
MachineMemOperand::Flags MMOFlags, |
--- |
8326 |
MachineMemOperand::Flags MMOFlags, |
--- |
| 8327 |
const AAMDNodes &AAInfo, const MDNode *Ranges) { |
--- |
8327 |
const AAMDNodes &AAInfo, const MDNode *Ranges) { |
--- |
| 8328 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
5 |
8328 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
5 |
| 8329 |
return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, |
5 |
8329 |
return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, |
5 |
| 8330 |
PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges); |
5 |
8330 |
PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges); |
5 |
| 8331 |
} |
--- |
8331 |
} |
--- |
| 8332 |
|
--- |
8332 |
|
--- |
| 8333 |
SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain, |
0 |
8333 |
SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain, |
0 |
| 8334 |
SDValue Ptr, MachineMemOperand *MMO) { |
--- |
8334 |
SDValue Ptr, MachineMemOperand *MMO) { |
--- |
| 8335 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
8335 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
| 8336 |
return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, |
0 |
8336 |
return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, |
0 |
| 8337 |
VT, MMO); |
0 |
8337 |
VT, MMO); |
0 |
| 8338 |
} |
--- |
8338 |
} |
--- |
| 8339 |
|
--- |
8339 |
|
--- |
| 8340 |
SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, |
0 |
8340 |
SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, |
0 |
| 8341 |
EVT VT, SDValue Chain, SDValue Ptr, |
--- |
8341 |
EVT VT, SDValue Chain, SDValue Ptr, |
--- |
| 8342 |
MachinePointerInfo PtrInfo, EVT MemVT, |
--- |
8342 |
MachinePointerInfo PtrInfo, EVT MemVT, |
--- |
| 8343 |
MaybeAlign Alignment, |
--- |
8343 |
MaybeAlign Alignment, |
--- |
| 8344 |
MachineMemOperand::Flags MMOFlags, |
--- |
8344 |
MachineMemOperand::Flags MMOFlags, |
--- |
| 8345 |
const AAMDNodes &AAInfo) { |
--- |
8345 |
const AAMDNodes &AAInfo) { |
--- |
| 8346 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
8346 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
| 8347 |
return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo, |
0 |
8347 |
return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo, |
0 |
| 8348 |
MemVT, Alignment, MMOFlags, AAInfo); |
0 |
8348 |
MemVT, Alignment, MMOFlags, AAInfo); |
0 |
| 8349 |
} |
--- |
8349 |
} |
--- |
| 8350 |
|
--- |
8350 |
|
--- |
| 8351 |
SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, |
0 |
8351 |
SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, |
0 |
| 8352 |
EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT, |
--- |
8352 |
EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT, |
--- |
| 8353 |
MachineMemOperand *MMO) { |
--- |
8353 |
MachineMemOperand *MMO) { |
--- |
| 8354 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
8354 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
| 8355 |
return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, |
0 |
8355 |
return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, |
0 |
| 8356 |
MemVT, MMO); |
0 |
8356 |
MemVT, MMO); |
0 |
| 8357 |
} |
--- |
8357 |
} |
--- |
| 8358 |
|
--- |
8358 |
|
--- |
| 8359 |
SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, |
0 |
8359 |
SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, |
0 |
| 8360 |
SDValue Base, SDValue Offset, |
--- |
8360 |
SDValue Base, SDValue Offset, |
--- |
| 8361 |
ISD::MemIndexedMode AM) { |
--- |
8361 |
ISD::MemIndexedMode AM) { |
--- |
| 8362 |
LoadSDNode *LD = cast(OrigLoad); |
0 |
8362 |
LoadSDNode *LD = cast(OrigLoad); |
0 |
| 8363 |
assert(LD->getOffset().isUndef() && "Load is already a indexed load!"); |
0 |
8363 |
assert(LD->getOffset().isUndef() && "Load is already a indexed load!"); |
0 |
| 8364 |
// Don't propagate the invariant or dereferenceable flags. |
--- |
8364 |
// Don't propagate the invariant or dereferenceable flags. |
--- |
| 8365 |
auto MMOFlags = |
--- |
8365 |
auto MMOFlags = |
--- |
| 8366 |
LD->getMemOperand()->getFlags() & |
0 |
8366 |
LD->getMemOperand()->getFlags() & |
0 |
| 8367 |
~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); |
0 |
8367 |
~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); |
0 |
| 8368 |
return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, |
0 |
8368 |
return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, |
0 |
| 8369 |
LD->getChain(), Base, Offset, LD->getPointerInfo(), |
0 |
8369 |
LD->getChain(), Base, Offset, LD->getPointerInfo(), |
0 |
| 8370 |
LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo()); |
0 |
8370 |
LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo()); |
0 |
| 8371 |
} |
--- |
8371 |
} |
--- |
| 8372 |
|
--- |
8372 |
|
--- |
| 8373 |
SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
4 |
8373 |
SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
4 |
| 8374 |
SDValue Ptr, MachinePointerInfo PtrInfo, |
--- |
8374 |
SDValue Ptr, MachinePointerInfo PtrInfo, |
--- |
| 8375 |
Align Alignment, |
--- |
8375 |
Align Alignment, |
--- |
| 8376 |
MachineMemOperand::Flags MMOFlags, |
--- |
8376 |
MachineMemOperand::Flags MMOFlags, |
--- |
| 8377 |
const AAMDNodes &AAInfo) { |
--- |
8377 |
const AAMDNodes &AAInfo) { |
--- |
| 8378 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
4 |
8378 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
4 |
| 8379 |
|
--- |
8379 |
|
--- |
| 8380 |
MMOFlags |= MachineMemOperand::MOStore; |
4 |
8380 |
MMOFlags |= MachineMemOperand::MOStore; |
4 |
| 8381 |
assert((MMOFlags & MachineMemOperand::MOLoad) == 0); |
4 |
8381 |
assert((MMOFlags & MachineMemOperand::MOLoad) == 0); |
4 |
| 8382 |
|
--- |
8382 |
|
--- |
| 8383 |
if (PtrInfo.V.isNull()) |
4 |
8383 |
if (PtrInfo.V.isNull()) |
4 |
| 8384 |
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr); |
0 |
8384 |
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr); |
0 |
| 8385 |
|
--- |
8385 |
|
--- |
| 8386 |
MachineFunction &MF = getMachineFunction(); |
4 |
8386 |
MachineFunction &MF = getMachineFunction(); |
4 |
| 8387 |
uint64_t Size = |
--- |
8387 |
uint64_t Size = |
--- |
| 8388 |
MemoryLocation::getSizeOrUnknown(Val.getValueType().getStoreSize()); |
4 |
8388 |
MemoryLocation::getSizeOrUnknown(Val.getValueType().getStoreSize()); |
4 |
| 8389 |
MachineMemOperand *MMO = |
--- |
8389 |
MachineMemOperand *MMO = |
--- |
| 8390 |
MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo); |
4 |
8390 |
MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo); |
4 |
| 8391 |
return getStore(Chain, dl, Val, Ptr, MMO); |
4 |
8391 |
return getStore(Chain, dl, Val, Ptr, MMO); |
4 |
| 8392 |
} |
--- |
8392 |
} |
--- |
| 8393 |
|
--- |
8393 |
|
--- |
| 8394 |
SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
4 |
8394 |
SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
4 |
| 8395 |
SDValue Ptr, MachineMemOperand *MMO) { |
--- |
8395 |
SDValue Ptr, MachineMemOperand *MMO) { |
--- |
| 8396 |
assert(Chain.getValueType() == MVT::Other && |
4 |
8396 |
assert(Chain.getValueType() == MVT::Other && |
4 |
| 8397 |
"Invalid chain type"); |
--- |
8397 |
"Invalid chain type"); |
--- |
| 8398 |
EVT VT = Val.getValueType(); |
4 |
8398 |
EVT VT = Val.getValueType(); |
4 |
| 8399 |
SDVTList VTs = getVTList(MVT::Other); |
4 |
8399 |
SDVTList VTs = getVTList(MVT::Other); |
4 |
| 8400 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
4 |
8400 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
4 |
| 8401 |
SDValue Ops[] = { Chain, Val, Ptr, Undef }; |
4 |
8401 |
SDValue Ops[] = { Chain, Val, Ptr, Undef }; |
4 |
| 8402 |
FoldingSetNodeID ID; |
4 |
8402 |
FoldingSetNodeID ID; |
4 |
| 8403 |
AddNodeIDNode(ID, ISD::STORE, VTs, Ops); |
4 |
8403 |
AddNodeIDNode(ID, ISD::STORE, VTs, Ops); |
4 |
| 8404 |
ID.AddInteger(VT.getRawBits()); |
4 |
8404 |
ID.AddInteger(VT.getRawBits()); |
4 |
| 8405 |
ID.AddInteger(getSyntheticNodeSubclassData( |
4 |
8405 |
ID.AddInteger(getSyntheticNodeSubclassData( |
4 |
| 8406 |
dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO)); |
4 |
8406 |
dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO)); |
4 |
| 8407 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
4 |
8407 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
4 |
| 8408 |
ID.AddInteger(MMO->getFlags()); |
4 |
8408 |
ID.AddInteger(MMO->getFlags()); |
4 |
| 8409 |
void *IP = nullptr; |
4 |
8409 |
void *IP = nullptr; |
4 |
| 8410 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
4 |
8410 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
4 |
| 8411 |
cast(E)->refineAlignment(MMO); |
0 |
8411 |
cast(E)->refineAlignment(MMO); |
0 |
| 8412 |
return SDValue(E, 0); |
0 |
8412 |
return SDValue(E, 0); |
0 |
| 8413 |
} |
--- |
8413 |
} |
--- |
| 8414 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, |
4 |
8414 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, |
4 |
| 8415 |
ISD::UNINDEXED, false, VT, MMO); |
4 |
8415 |
ISD::UNINDEXED, false, VT, MMO); |
4 |
| 8416 |
createOperands(N, Ops); |
4 |
8416 |
createOperands(N, Ops); |
4 |
| 8417 |
|
--- |
8417 |
|
--- |
| 8418 |
CSEMap.InsertNode(N, IP); |
4 |
8418 |
CSEMap.InsertNode(N, IP); |
4 |
| 8419 |
InsertNode(N); |
4 |
8419 |
InsertNode(N); |
4 |
| 8420 |
SDValue V(N, 0); |
4 |
8420 |
SDValue V(N, 0); |
4 |
| 8421 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
4 |
8421 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
4 |
| 8422 |
return V; |
4 |
8422 |
return V; |
4 |
| 8423 |
} |
4 |
8423 |
} |
4 |
| 8424 |
|
--- |
8424 |
|
--- |
| 8425 |
SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
0 |
8425 |
SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
0 |
| 8426 |
SDValue Ptr, MachinePointerInfo PtrInfo, |
--- |
8426 |
SDValue Ptr, MachinePointerInfo PtrInfo, |
--- |
| 8427 |
EVT SVT, Align Alignment, |
--- |
8427 |
EVT SVT, Align Alignment, |
--- |
| 8428 |
MachineMemOperand::Flags MMOFlags, |
--- |
8428 |
MachineMemOperand::Flags MMOFlags, |
--- |
| 8429 |
const AAMDNodes &AAInfo) { |
--- |
8429 |
const AAMDNodes &AAInfo) { |
--- |
| 8430 |
assert(Chain.getValueType() == MVT::Other && |
0 |
8430 |
assert(Chain.getValueType() == MVT::Other && |
0 |
| 8431 |
"Invalid chain type"); |
--- |
8431 |
"Invalid chain type"); |
--- |
| 8432 |
|
--- |
8432 |
|
--- |
| 8433 |
MMOFlags |= MachineMemOperand::MOStore; |
0 |
8433 |
MMOFlags |= MachineMemOperand::MOStore; |
0 |
| 8434 |
assert((MMOFlags & MachineMemOperand::MOLoad) == 0); |
0 |
8434 |
assert((MMOFlags & MachineMemOperand::MOLoad) == 0); |
0 |
| 8435 |
|
--- |
8435 |
|
--- |
| 8436 |
if (PtrInfo.V.isNull()) |
0 |
8436 |
if (PtrInfo.V.isNull()) |
0 |
| 8437 |
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr); |
0 |
8437 |
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr); |
0 |
| 8438 |
|
--- |
8438 |
|
--- |
| 8439 |
MachineFunction &MF = getMachineFunction(); |
0 |
8439 |
MachineFunction &MF = getMachineFunction(); |
0 |
| 8440 |
MachineMemOperand *MMO = MF.getMachineMemOperand( |
0 |
8440 |
MachineMemOperand *MMO = MF.getMachineMemOperand( |
0 |
| 8441 |
PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()), |
0 |
8441 |
PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()), |
0 |
| 8442 |
Alignment, AAInfo); |
--- |
8442 |
Alignment, AAInfo); |
--- |
| 8443 |
return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); |
0 |
8443 |
return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); |
0 |
| 8444 |
} |
--- |
8444 |
} |
--- |
| 8445 |
|
--- |
8445 |
|
--- |
| 8446 |
SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
0 |
8446 |
SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
0 |
| 8447 |
SDValue Ptr, EVT SVT, |
--- |
8447 |
SDValue Ptr, EVT SVT, |
--- |
| 8448 |
MachineMemOperand *MMO) { |
--- |
8448 |
MachineMemOperand *MMO) { |
--- |
| 8449 |
EVT VT = Val.getValueType(); |
0 |
8449 |
EVT VT = Val.getValueType(); |
0 |
| 8450 |
|
--- |
8450 |
|
--- |
| 8451 |
assert(Chain.getValueType() == MVT::Other && |
0 |
8451 |
assert(Chain.getValueType() == MVT::Other && |
0 |
| 8452 |
"Invalid chain type"); |
--- |
8452 |
"Invalid chain type"); |
--- |
| 8453 |
if (VT == SVT) |
0 |
8453 |
if (VT == SVT) |
0 |
| 8454 |
return getStore(Chain, dl, Val, Ptr, MMO); |
0 |
8454 |
return getStore(Chain, dl, Val, Ptr, MMO); |
0 |
| 8455 |
|
--- |
8455 |
|
--- |
| 8456 |
assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && |
0 |
8456 |
assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && |
0 |
| 8457 |
"Should only be a truncating store, not extending!"); |
--- |
8457 |
"Should only be a truncating store, not extending!"); |
--- |
| 8458 |
assert(VT.isInteger() == SVT.isInteger() && |
0 |
8458 |
assert(VT.isInteger() == SVT.isInteger() && |
0 |
| 8459 |
"Can't do FP-INT conversion!"); |
--- |
8459 |
"Can't do FP-INT conversion!"); |
--- |
| 8460 |
assert(VT.isVector() == SVT.isVector() && |
0 |
8460 |
assert(VT.isVector() == SVT.isVector() && |
0 |
| 8461 |
"Cannot use trunc store to convert to or from a vector!"); |
--- |
8461 |
"Cannot use trunc store to convert to or from a vector!"); |
--- |
| 8462 |
assert((!VT.isVector() || |
0 |
8462 |
assert((!VT.isVector() || |
0 |
| 8463 |
VT.getVectorElementCount() == SVT.getVectorElementCount()) && |
--- |
8463 |
VT.getVectorElementCount() == SVT.getVectorElementCount()) && |
--- |
| 8464 |
"Cannot use trunc store to change the number of vector elements!"); |
--- |
8464 |
"Cannot use trunc store to change the number of vector elements!"); |
--- |
| 8465 |
|
--- |
8465 |
|
--- |
| 8466 |
SDVTList VTs = getVTList(MVT::Other); |
0 |
8466 |
SDVTList VTs = getVTList(MVT::Other); |
0 |
| 8467 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
8467 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
| 8468 |
SDValue Ops[] = { Chain, Val, Ptr, Undef }; |
0 |
8468 |
SDValue Ops[] = { Chain, Val, Ptr, Undef }; |
0 |
| 8469 |
FoldingSetNodeID ID; |
0 |
8469 |
FoldingSetNodeID ID; |
0 |
| 8470 |
AddNodeIDNode(ID, ISD::STORE, VTs, Ops); |
0 |
8470 |
AddNodeIDNode(ID, ISD::STORE, VTs, Ops); |
0 |
| 8471 |
ID.AddInteger(SVT.getRawBits()); |
0 |
8471 |
ID.AddInteger(SVT.getRawBits()); |
0 |
| 8472 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
8472 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
| 8473 |
dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO)); |
0 |
8473 |
dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO)); |
0 |
| 8474 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
8474 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
| 8475 |
ID.AddInteger(MMO->getFlags()); |
0 |
8475 |
ID.AddInteger(MMO->getFlags()); |
0 |
| 8476 |
void *IP = nullptr; |
0 |
8476 |
void *IP = nullptr; |
0 |
| 8477 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
8477 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
| 8478 |
cast(E)->refineAlignment(MMO); |
0 |
8478 |
cast(E)->refineAlignment(MMO); |
0 |
| 8479 |
return SDValue(E, 0); |
0 |
8479 |
return SDValue(E, 0); |
0 |
| 8480 |
} |
--- |
8480 |
} |
--- |
| 8481 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, |
0 |
8481 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, |
0 |
| 8482 |
ISD::UNINDEXED, true, SVT, MMO); |
0 |
8482 |
ISD::UNINDEXED, true, SVT, MMO); |
0 |
| 8483 |
createOperands(N, Ops); |
0 |
8483 |
createOperands(N, Ops); |
0 |
| 8484 |
|
--- |
8484 |
|
--- |
| 8485 |
CSEMap.InsertNode(N, IP); |
0 |
8485 |
CSEMap.InsertNode(N, IP); |
0 |
| 8486 |
InsertNode(N); |
0 |
8486 |
InsertNode(N); |
0 |
| 8487 |
SDValue V(N, 0); |
0 |
8487 |
SDValue V(N, 0); |
0 |
| 8488 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
8488 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 8489 |
return V; |
0 |
8489 |
return V; |
0 |
| 8490 |
} |
0 |
8490 |
} |
0 |
| 8491 |
|
--- |
8491 |
|
--- |
| 8492 |
SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl, |
0 |
8492 |
SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl, |
0 |
| 8493 |
SDValue Base, SDValue Offset, |
--- |
8493 |
SDValue Base, SDValue Offset, |
--- |
| 8494 |
ISD::MemIndexedMode AM) { |
--- |
8494 |
ISD::MemIndexedMode AM) { |
--- |
| 8495 |
StoreSDNode *ST = cast(OrigStore); |
0 |
8495 |
StoreSDNode *ST = cast(OrigStore); |
0 |
| 8496 |
assert(ST->getOffset().isUndef() && "Store is already a indexed store!"); |
0 |
8496 |
assert(ST->getOffset().isUndef() && "Store is already a indexed store!"); |
0 |
| 8497 |
SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); |
0 |
8497 |
SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); |
0 |
| 8498 |
SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset }; |
0 |
8498 |
SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset }; |
0 |
| 8499 |
FoldingSetNodeID ID; |
0 |
8499 |
FoldingSetNodeID ID; |
0 |
| 8500 |
AddNodeIDNode(ID, ISD::STORE, VTs, Ops); |
0 |
8500 |
AddNodeIDNode(ID, ISD::STORE, VTs, Ops); |
0 |
| 8501 |
ID.AddInteger(ST->getMemoryVT().getRawBits()); |
0 |
8501 |
ID.AddInteger(ST->getMemoryVT().getRawBits()); |
0 |
| 8502 |
ID.AddInteger(ST->getRawSubclassData()); |
0 |
8502 |
ID.AddInteger(ST->getRawSubclassData()); |
0 |
| 8503 |
ID.AddInteger(ST->getPointerInfo().getAddrSpace()); |
0 |
8503 |
ID.AddInteger(ST->getPointerInfo().getAddrSpace()); |
0 |
| 8504 |
ID.AddInteger(ST->getMemOperand()->getFlags()); |
0 |
8504 |
ID.AddInteger(ST->getMemOperand()->getFlags()); |
0 |
| 8505 |
void *IP = nullptr; |
0 |
8505 |
void *IP = nullptr; |
0 |
| 8506 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) |
0 |
8506 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) |
0 |
| 8507 |
return SDValue(E, 0); |
0 |
8507 |
return SDValue(E, 0); |
0 |
| 8508 |
|
--- |
8508 |
|
--- |
| 8509 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, |
0 |
8509 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, |
0 |
| 8510 |
ST->isTruncatingStore(), ST->getMemoryVT(), |
0 |
8510 |
ST->isTruncatingStore(), ST->getMemoryVT(), |
0 |
| 8511 |
ST->getMemOperand()); |
0 |
8511 |
ST->getMemOperand()); |
0 |
| 8512 |
createOperands(N, Ops); |
0 |
8512 |
createOperands(N, Ops); |
0 |
| 8513 |
|
--- |
8513 |
|
--- |
| 8514 |
CSEMap.InsertNode(N, IP); |
0 |
8514 |
CSEMap.InsertNode(N, IP); |
0 |
| 8515 |
InsertNode(N); |
0 |
8515 |
InsertNode(N); |
0 |
| 8516 |
SDValue V(N, 0); |
0 |
8516 |
SDValue V(N, 0); |
0 |
| 8517 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
8517 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 8518 |
return V; |
0 |
8518 |
return V; |
0 |
| 8519 |
} |
0 |
8519 |
} |
0 |
| 8520 |
|
--- |
8520 |
|
--- |
| 8521 |
SDValue SelectionDAG::getLoadVP( |
0 |
8521 |
SDValue SelectionDAG::getLoadVP( |
0 |
| 8522 |
ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, |
--- |
8522 |
ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, |
--- |
| 8523 |
SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, |
--- |
8523 |
SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, |
--- |
| 8524 |
MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, |
--- |
8524 |
MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, |
--- |
| 8525 |
MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, |
--- |
8525 |
MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, |
--- |
| 8526 |
const MDNode *Ranges, bool IsExpanding) { |
--- |
8526 |
const MDNode *Ranges, bool IsExpanding) { |
--- |
| 8527 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
8527 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
| 8528 |
|
--- |
8528 |
|
--- |
| 8529 |
MMOFlags |= MachineMemOperand::MOLoad; |
0 |
8529 |
MMOFlags |= MachineMemOperand::MOLoad; |
0 |
| 8530 |
assert((MMOFlags & MachineMemOperand::MOStore) == 0); |
0 |
8530 |
assert((MMOFlags & MachineMemOperand::MOStore) == 0); |
0 |
| 8531 |
// If we don't have a PtrInfo, infer the trivial frame index case to simplify |
--- |
8531 |
// If we don't have a PtrInfo, infer the trivial frame index case to simplify |
--- |
| 8532 |
// clients. |
--- |
8532 |
// clients. |
--- |
| 8533 |
if (PtrInfo.V.isNull()) |
0 |
8533 |
if (PtrInfo.V.isNull()) |
0 |
| 8534 |
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset); |
0 |
8534 |
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset); |
0 |
| 8535 |
|
--- |
8535 |
|
--- |
| 8536 |
uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize()); |
0 |
8536 |
uint64_t Size = MemoryLocation::getSizeOrUnknown(MemVT.getStoreSize()); |
0 |
| 8537 |
MachineFunction &MF = getMachineFunction(); |
0 |
8537 |
MachineFunction &MF = getMachineFunction(); |
0 |
| 8538 |
MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, |
0 |
8538 |
MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, |
0 |
| 8539 |
Alignment, AAInfo, Ranges); |
--- |
8539 |
Alignment, AAInfo, Ranges); |
--- |
| 8540 |
return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT, |
0 |
8540 |
return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT, |
0 |
| 8541 |
MMO, IsExpanding); |
0 |
8541 |
MMO, IsExpanding); |
0 |
| 8542 |
} |
--- |
8542 |
} |
--- |
| 8543 |
|
--- |
8543 |
|
--- |
| 8544 |
SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM, |
0 |
8544 |
SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM, |
0 |
| 8545 |
ISD::LoadExtType ExtType, EVT VT, |
--- |
8545 |
ISD::LoadExtType ExtType, EVT VT, |
--- |
| 8546 |
const SDLoc &dl, SDValue Chain, SDValue Ptr, |
--- |
8546 |
const SDLoc &dl, SDValue Chain, SDValue Ptr, |
--- |
| 8547 |
SDValue Offset, SDValue Mask, SDValue EVL, |
--- |
8547 |
SDValue Offset, SDValue Mask, SDValue EVL, |
--- |
| 8548 |
EVT MemVT, MachineMemOperand *MMO, |
--- |
8548 |
EVT MemVT, MachineMemOperand *MMO, |
--- |
| 8549 |
bool IsExpanding) { |
--- |
8549 |
bool IsExpanding) { |
--- |
| 8550 |
bool Indexed = AM != ISD::UNINDEXED; |
0 |
8550 |
bool Indexed = AM != ISD::UNINDEXED; |
0 |
| 8551 |
assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!"); |
0 |
8551 |
assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!"); |
0 |
| 8552 |
|
--- |
8552 |
|
--- |
| 8553 |
SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other) |
0 |
8553 |
SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other) |
0 |
| 8554 |
: getVTList(VT, MVT::Other); |
0 |
8554 |
: getVTList(VT, MVT::Other); |
0 |
| 8555 |
SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL}; |
0 |
8555 |
SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL}; |
0 |
| 8556 |
FoldingSetNodeID ID; |
0 |
8556 |
FoldingSetNodeID ID; |
0 |
| 8557 |
AddNodeIDNode(ID, ISD::VP_LOAD, VTs, Ops); |
0 |
8557 |
AddNodeIDNode(ID, ISD::VP_LOAD, VTs, Ops); |
0 |
| 8558 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
8558 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
| 8559 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
8559 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
| 8560 |
dl.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO)); |
--- |
8560 |
dl.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO)); |
--- |
| 8561 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
8561 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
| 8562 |
ID.AddInteger(MMO->getFlags()); |
0 |
8562 |
ID.AddInteger(MMO->getFlags()); |
0 |
| 8563 |
void *IP = nullptr; |
0 |
8563 |
void *IP = nullptr; |
0 |
| 8564 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
8564 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
| 8565 |
cast(E)->refineAlignment(MMO); |
0 |
8565 |
cast(E)->refineAlignment(MMO); |
0 |
| 8566 |
return SDValue(E, 0); |
0 |
8566 |
return SDValue(E, 0); |
0 |
| 8567 |
} |
--- |
8567 |
} |
--- |
| 8568 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, |
0 |
8568 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, |
0 |
| 8569 |
ExtType, IsExpanding, MemVT, MMO); |
--- |
8569 |
ExtType, IsExpanding, MemVT, MMO); |
--- |
| 8570 |
createOperands(N, Ops); |
0 |
8570 |
createOperands(N, Ops); |
0 |
| 8571 |
|
--- |
8571 |
|
--- |
| 8572 |
CSEMap.InsertNode(N, IP); |
0 |
8572 |
CSEMap.InsertNode(N, IP); |
0 |
| 8573 |
InsertNode(N); |
0 |
8573 |
InsertNode(N); |
0 |
| 8574 |
SDValue V(N, 0); |
0 |
8574 |
SDValue V(N, 0); |
0 |
| 8575 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
8575 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 8576 |
return V; |
0 |
8576 |
return V; |
0 |
| 8577 |
} |
0 |
8577 |
} |
0 |
| 8578 |
|
--- |
8578 |
|
--- |
| 8579 |
SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain, |
0 |
8579 |
SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain, |
0 |
| 8580 |
SDValue Ptr, SDValue Mask, SDValue EVL, |
--- |
8580 |
SDValue Ptr, SDValue Mask, SDValue EVL, |
--- |
| 8581 |
MachinePointerInfo PtrInfo, |
--- |
8581 |
MachinePointerInfo PtrInfo, |
--- |
| 8582 |
MaybeAlign Alignment, |
--- |
8582 |
MaybeAlign Alignment, |
--- |
| 8583 |
MachineMemOperand::Flags MMOFlags, |
--- |
8583 |
MachineMemOperand::Flags MMOFlags, |
--- |
| 8584 |
const AAMDNodes &AAInfo, const MDNode *Ranges, |
--- |
8584 |
const AAMDNodes &AAInfo, const MDNode *Ranges, |
--- |
| 8585 |
bool IsExpanding) { |
--- |
8585 |
bool IsExpanding) { |
--- |
| 8586 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
8586 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
| 8587 |
return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, |
0 |
8587 |
return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, |
0 |
| 8588 |
Mask, EVL, PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges, |
--- |
8588 |
Mask, EVL, PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges, |
--- |
| 8589 |
IsExpanding); |
0 |
8589 |
IsExpanding); |
0 |
| 8590 |
} |
--- |
8590 |
} |
--- |
| 8591 |
|
--- |
8591 |
|
--- |
| 8592 |
SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain, |
0 |
8592 |
SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain, |
0 |
| 8593 |
SDValue Ptr, SDValue Mask, SDValue EVL, |
--- |
8593 |
SDValue Ptr, SDValue Mask, SDValue EVL, |
--- |
| 8594 |
MachineMemOperand *MMO, bool IsExpanding) { |
--- |
8594 |
MachineMemOperand *MMO, bool IsExpanding) { |
--- |
| 8595 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
8595 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
| 8596 |
return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, |
0 |
8596 |
return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, |
0 |
| 8597 |
Mask, EVL, VT, MMO, IsExpanding); |
0 |
8597 |
Mask, EVL, VT, MMO, IsExpanding); |
0 |
| 8598 |
} |
--- |
8598 |
} |
--- |
| 8599 |
|
--- |
8599 |
|
--- |
| 8600 |
SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, |
0 |
8600 |
SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, |
0 |
| 8601 |
EVT VT, SDValue Chain, SDValue Ptr, |
--- |
8601 |
EVT VT, SDValue Chain, SDValue Ptr, |
--- |
| 8602 |
SDValue Mask, SDValue EVL, |
--- |
8602 |
SDValue Mask, SDValue EVL, |
--- |
| 8603 |
MachinePointerInfo PtrInfo, EVT MemVT, |
--- |
8603 |
MachinePointerInfo PtrInfo, EVT MemVT, |
--- |
| 8604 |
MaybeAlign Alignment, |
--- |
8604 |
MaybeAlign Alignment, |
--- |
| 8605 |
MachineMemOperand::Flags MMOFlags, |
--- |
8605 |
MachineMemOperand::Flags MMOFlags, |
--- |
| 8606 |
const AAMDNodes &AAInfo, bool IsExpanding) { |
--- |
8606 |
const AAMDNodes &AAInfo, bool IsExpanding) { |
--- |
| 8607 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
8607 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
| 8608 |
return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask, |
0 |
8608 |
return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask, |
0 |
| 8609 |
EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, nullptr, |
--- |
8609 |
EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, nullptr, |
--- |
| 8610 |
IsExpanding); |
0 |
8610 |
IsExpanding); |
0 |
| 8611 |
} |
--- |
8611 |
} |
--- |
| 8612 |
|
--- |
8612 |
|
--- |
| 8613 |
SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, |
0 |
8613 |
SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, |
0 |
| 8614 |
EVT VT, SDValue Chain, SDValue Ptr, |
--- |
8614 |
EVT VT, SDValue Chain, SDValue Ptr, |
--- |
| 8615 |
SDValue Mask, SDValue EVL, EVT MemVT, |
--- |
8615 |
SDValue Mask, SDValue EVL, EVT MemVT, |
--- |
| 8616 |
MachineMemOperand *MMO, bool IsExpanding) { |
--- |
8616 |
MachineMemOperand *MMO, bool IsExpanding) { |
--- |
| 8617 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
8617 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
| 8618 |
return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask, |
0 |
8618 |
return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask, |
0 |
| 8619 |
EVL, MemVT, MMO, IsExpanding); |
0 |
8619 |
EVL, MemVT, MMO, IsExpanding); |
0 |
| 8620 |
} |
--- |
8620 |
} |
--- |
| 8621 |
|
--- |
8621 |
|
--- |
| 8622 |
SDValue SelectionDAG::getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl, |
0 |
8622 |
SDValue SelectionDAG::getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl, |
0 |
| 8623 |
SDValue Base, SDValue Offset, |
--- |
8623 |
SDValue Base, SDValue Offset, |
--- |
| 8624 |
ISD::MemIndexedMode AM) { |
--- |
8624 |
ISD::MemIndexedMode AM) { |
--- |
| 8625 |
auto *LD = cast(OrigLoad); |
0 |
8625 |
auto *LD = cast(OrigLoad); |
0 |
| 8626 |
assert(LD->getOffset().isUndef() && "Load is already a indexed load!"); |
0 |
8626 |
assert(LD->getOffset().isUndef() && "Load is already a indexed load!"); |
0 |
| 8627 |
// Don't propagate the invariant or dereferenceable flags. |
--- |
8627 |
// Don't propagate the invariant or dereferenceable flags. |
--- |
| 8628 |
auto MMOFlags = |
--- |
8628 |
auto MMOFlags = |
--- |
| 8629 |
LD->getMemOperand()->getFlags() & |
0 |
8629 |
LD->getMemOperand()->getFlags() & |
0 |
| 8630 |
~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); |
0 |
8630 |
~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); |
0 |
| 8631 |
return getLoadVP(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, |
0 |
8631 |
return getLoadVP(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, |
0 |
| 8632 |
LD->getChain(), Base, Offset, LD->getMask(), |
0 |
8632 |
LD->getChain(), Base, Offset, LD->getMask(), |
0 |
| 8633 |
LD->getVectorLength(), LD->getPointerInfo(), |
0 |
8633 |
LD->getVectorLength(), LD->getPointerInfo(), |
0 |
| 8634 |
LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo(), |
0 |
8634 |
LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo(), |
0 |
| 8635 |
nullptr, LD->isExpandingLoad()); |
0 |
8635 |
nullptr, LD->isExpandingLoad()); |
0 |
| 8636 |
} |
--- |
8636 |
} |
--- |
| 8637 |
|
--- |
8637 |
|
--- |
| 8638 |
SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, |
0 |
8638 |
SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, |
0 |
| 8639 |
SDValue Ptr, SDValue Offset, SDValue Mask, |
--- |
8639 |
SDValue Ptr, SDValue Offset, SDValue Mask, |
--- |
| 8640 |
SDValue EVL, EVT MemVT, MachineMemOperand *MMO, |
--- |
8640 |
SDValue EVL, EVT MemVT, MachineMemOperand *MMO, |
--- |
| 8641 |
ISD::MemIndexedMode AM, bool IsTruncating, |
--- |
8641 |
ISD::MemIndexedMode AM, bool IsTruncating, |
--- |
| 8642 |
bool IsCompressing) { |
--- |
8642 |
bool IsCompressing) { |
--- |
| 8643 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
8643 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
| 8644 |
bool Indexed = AM != ISD::UNINDEXED; |
0 |
8644 |
bool Indexed = AM != ISD::UNINDEXED; |
0 |
| 8645 |
assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!"); |
0 |
8645 |
assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!"); |
0 |
| 8646 |
SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other) |
0 |
8646 |
SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other) |
0 |
| 8647 |
: getVTList(MVT::Other); |
0 |
8647 |
: getVTList(MVT::Other); |
0 |
| 8648 |
SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL}; |
0 |
8648 |
SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL}; |
0 |
| 8649 |
FoldingSetNodeID ID; |
0 |
8649 |
FoldingSetNodeID ID; |
0 |
| 8650 |
AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops); |
0 |
8650 |
AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops); |
0 |
| 8651 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
8651 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
| 8652 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
8652 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
| 8653 |
dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO)); |
--- |
8653 |
dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO)); |
--- |
| 8654 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
8654 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
| 8655 |
ID.AddInteger(MMO->getFlags()); |
0 |
8655 |
ID.AddInteger(MMO->getFlags()); |
0 |
| 8656 |
void *IP = nullptr; |
0 |
8656 |
void *IP = nullptr; |
0 |
| 8657 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
8657 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
| 8658 |
cast(E)->refineAlignment(MMO); |
0 |
8658 |
cast(E)->refineAlignment(MMO); |
0 |
| 8659 |
return SDValue(E, 0); |
0 |
8659 |
return SDValue(E, 0); |
0 |
| 8660 |
} |
--- |
8660 |
} |
--- |
| 8661 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, |
0 |
8661 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, |
0 |
| 8662 |
IsTruncating, IsCompressing, MemVT, MMO); |
--- |
8662 |
IsTruncating, IsCompressing, MemVT, MMO); |
--- |
| 8663 |
createOperands(N, Ops); |
0 |
8663 |
createOperands(N, Ops); |
0 |
| 8664 |
|
--- |
8664 |
|
--- |
| 8665 |
CSEMap.InsertNode(N, IP); |
0 |
8665 |
CSEMap.InsertNode(N, IP); |
0 |
| 8666 |
InsertNode(N); |
0 |
8666 |
InsertNode(N); |
0 |
| 8667 |
SDValue V(N, 0); |
0 |
8667 |
SDValue V(N, 0); |
0 |
| 8668 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
8668 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 8669 |
return V; |
0 |
8669 |
return V; |
0 |
| 8670 |
} |
0 |
8670 |
} |
0 |
| 8671 |
|
--- |
8671 |
|
--- |
| 8672 |
SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl, |
0 |
8672 |
SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl, |
0 |
| 8673 |
SDValue Val, SDValue Ptr, SDValue Mask, |
--- |
8673 |
SDValue Val, SDValue Ptr, SDValue Mask, |
--- |
| 8674 |
SDValue EVL, MachinePointerInfo PtrInfo, |
--- |
8674 |
SDValue EVL, MachinePointerInfo PtrInfo, |
--- |
| 8675 |
EVT SVT, Align Alignment, |
--- |
8675 |
EVT SVT, Align Alignment, |
--- |
| 8676 |
MachineMemOperand::Flags MMOFlags, |
--- |
8676 |
MachineMemOperand::Flags MMOFlags, |
--- |
| 8677 |
const AAMDNodes &AAInfo, |
--- |
8677 |
const AAMDNodes &AAInfo, |
--- |
| 8678 |
bool IsCompressing) { |
--- |
8678 |
bool IsCompressing) { |
--- |
| 8679 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
8679 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
| 8680 |
|
--- |
8680 |
|
--- |
| 8681 |
MMOFlags |= MachineMemOperand::MOStore; |
0 |
8681 |
MMOFlags |= MachineMemOperand::MOStore; |
0 |
| 8682 |
assert((MMOFlags & MachineMemOperand::MOLoad) == 0); |
0 |
8682 |
assert((MMOFlags & MachineMemOperand::MOLoad) == 0); |
0 |
| 8683 |
|
--- |
8683 |
|
--- |
| 8684 |
if (PtrInfo.V.isNull()) |
0 |
8684 |
if (PtrInfo.V.isNull()) |
0 |
| 8685 |
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr); |
0 |
8685 |
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr); |
0 |
| 8686 |
|
--- |
8686 |
|
--- |
| 8687 |
MachineFunction &MF = getMachineFunction(); |
0 |
8687 |
MachineFunction &MF = getMachineFunction(); |
0 |
| 8688 |
MachineMemOperand *MMO = MF.getMachineMemOperand( |
0 |
8688 |
MachineMemOperand *MMO = MF.getMachineMemOperand( |
0 |
| 8689 |
PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()), |
0 |
8689 |
PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()), |
0 |
| 8690 |
Alignment, AAInfo); |
--- |
8690 |
Alignment, AAInfo); |
--- |
| 8691 |
return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO, |
0 |
8691 |
return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO, |
0 |
| 8692 |
IsCompressing); |
0 |
8692 |
IsCompressing); |
0 |
| 8693 |
} |
--- |
8693 |
} |
--- |
| 8694 |
|
--- |
8694 |
|
--- |
| 8695 |
SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl, |
0 |
8695 |
SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl, |
0 |
| 8696 |
SDValue Val, SDValue Ptr, SDValue Mask, |
--- |
8696 |
SDValue Val, SDValue Ptr, SDValue Mask, |
--- |
| 8697 |
SDValue EVL, EVT SVT, |
--- |
8697 |
SDValue EVL, EVT SVT, |
--- |
| 8698 |
MachineMemOperand *MMO, |
--- |
8698 |
MachineMemOperand *MMO, |
--- |
| 8699 |
bool IsCompressing) { |
--- |
8699 |
bool IsCompressing) { |
--- |
| 8700 |
EVT VT = Val.getValueType(); |
0 |
8700 |
EVT VT = Val.getValueType(); |
0 |
| 8701 |
|
--- |
8701 |
|
--- |
| 8702 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
8702 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
| 8703 |
if (VT == SVT) |
0 |
8703 |
if (VT == SVT) |
0 |
| 8704 |
return getStoreVP(Chain, dl, Val, Ptr, getUNDEF(Ptr.getValueType()), Mask, |
0 |
8704 |
return getStoreVP(Chain, dl, Val, Ptr, getUNDEF(Ptr.getValueType()), Mask, |
0 |
| 8705 |
EVL, VT, MMO, ISD::UNINDEXED, |
--- |
8705 |
EVL, VT, MMO, ISD::UNINDEXED, |
--- |
| 8706 |
/*IsTruncating*/ false, IsCompressing); |
0 |
8706 |
/*IsTruncating*/ false, IsCompressing); |
0 |
| 8707 |
|
--- |
8707 |
|
--- |
| 8708 |
assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && |
0 |
8708 |
assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && |
0 |
| 8709 |
"Should only be a truncating store, not extending!"); |
--- |
8709 |
"Should only be a truncating store, not extending!"); |
--- |
| 8710 |
assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!"); |
0 |
8710 |
assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!"); |
0 |
| 8711 |
assert(VT.isVector() == SVT.isVector() && |
0 |
8711 |
assert(VT.isVector() == SVT.isVector() && |
0 |
| 8712 |
"Cannot use trunc store to convert to or from a vector!"); |
--- |
8712 |
"Cannot use trunc store to convert to or from a vector!"); |
--- |
| 8713 |
assert((!VT.isVector() || |
0 |
8713 |
assert((!VT.isVector() || |
0 |
| 8714 |
VT.getVectorElementCount() == SVT.getVectorElementCount()) && |
--- |
8714 |
VT.getVectorElementCount() == SVT.getVectorElementCount()) && |
--- |
| 8715 |
"Cannot use trunc store to change the number of vector elements!"); |
--- |
8715 |
"Cannot use trunc store to change the number of vector elements!"); |
--- |
| 8716 |
|
--- |
8716 |
|
--- |
| 8717 |
SDVTList VTs = getVTList(MVT::Other); |
0 |
8717 |
SDVTList VTs = getVTList(MVT::Other); |
0 |
| 8718 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
8718 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
| 8719 |
SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL}; |
0 |
8719 |
SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL}; |
0 |
| 8720 |
FoldingSetNodeID ID; |
0 |
8720 |
FoldingSetNodeID ID; |
0 |
| 8721 |
AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops); |
0 |
8721 |
AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops); |
0 |
| 8722 |
ID.AddInteger(SVT.getRawBits()); |
0 |
8722 |
ID.AddInteger(SVT.getRawBits()); |
0 |
| 8723 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
8723 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
| 8724 |
dl.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO)); |
0 |
8724 |
dl.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO)); |
0 |
| 8725 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
8725 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
| 8726 |
ID.AddInteger(MMO->getFlags()); |
0 |
8726 |
ID.AddInteger(MMO->getFlags()); |
0 |
| 8727 |
void *IP = nullptr; |
0 |
8727 |
void *IP = nullptr; |
0 |
| 8728 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
8728 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
| 8729 |
cast(E)->refineAlignment(MMO); |
0 |
8729 |
cast(E)->refineAlignment(MMO); |
0 |
| 8730 |
return SDValue(E, 0); |
0 |
8730 |
return SDValue(E, 0); |
0 |
| 8731 |
} |
--- |
8731 |
} |
--- |
| 8732 |
auto *N = |
--- |
8732 |
auto *N = |
--- |
| 8733 |
newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, |
0 |
8733 |
newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, |
0 |
| 8734 |
ISD::UNINDEXED, true, IsCompressing, SVT, MMO); |
0 |
8734 |
ISD::UNINDEXED, true, IsCompressing, SVT, MMO); |
0 |
| 8735 |
createOperands(N, Ops); |
0 |
8735 |
createOperands(N, Ops); |
0 |
| 8736 |
|
--- |
8736 |
|
--- |
| 8737 |
CSEMap.InsertNode(N, IP); |
0 |
8737 |
CSEMap.InsertNode(N, IP); |
0 |
| 8738 |
InsertNode(N); |
0 |
8738 |
InsertNode(N); |
0 |
| 8739 |
SDValue V(N, 0); |
0 |
8739 |
SDValue V(N, 0); |
0 |
| 8740 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
8740 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 8741 |
return V; |
0 |
8741 |
return V; |
0 |
| 8742 |
} |
0 |
8742 |
} |
0 |
| 8743 |
|
--- |
8743 |
|
--- |
| 8744 |
SDValue SelectionDAG::getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl, |
0 |
8744 |
SDValue SelectionDAG::getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl, |
0 |
| 8745 |
SDValue Base, SDValue Offset, |
--- |
8745 |
SDValue Base, SDValue Offset, |
--- |
| 8746 |
ISD::MemIndexedMode AM) { |
--- |
8746 |
ISD::MemIndexedMode AM) { |
--- |
| 8747 |
auto *ST = cast(OrigStore); |
0 |
8747 |
auto *ST = cast(OrigStore); |
0 |
| 8748 |
assert(ST->getOffset().isUndef() && "Store is already an indexed store!"); |
0 |
8748 |
assert(ST->getOffset().isUndef() && "Store is already an indexed store!"); |
0 |
| 8749 |
SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); |
0 |
8749 |
SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); |
0 |
| 8750 |
SDValue Ops[] = {ST->getChain(), ST->getValue(), Base, |
0 |
8750 |
SDValue Ops[] = {ST->getChain(), ST->getValue(), Base, |
0 |
| 8751 |
Offset, ST->getMask(), ST->getVectorLength()}; |
0 |
8751 |
Offset, ST->getMask(), ST->getVectorLength()}; |
0 |
| 8752 |
FoldingSetNodeID ID; |
0 |
8752 |
FoldingSetNodeID ID; |
0 |
| 8753 |
AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops); |
0 |
8753 |
AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops); |
0 |
| 8754 |
ID.AddInteger(ST->getMemoryVT().getRawBits()); |
0 |
8754 |
ID.AddInteger(ST->getMemoryVT().getRawBits()); |
0 |
| 8755 |
ID.AddInteger(ST->getRawSubclassData()); |
0 |
8755 |
ID.AddInteger(ST->getRawSubclassData()); |
0 |
| 8756 |
ID.AddInteger(ST->getPointerInfo().getAddrSpace()); |
0 |
8756 |
ID.AddInteger(ST->getPointerInfo().getAddrSpace()); |
0 |
| 8757 |
ID.AddInteger(ST->getMemOperand()->getFlags()); |
0 |
8757 |
ID.AddInteger(ST->getMemOperand()->getFlags()); |
0 |
| 8758 |
void *IP = nullptr; |
0 |
8758 |
void *IP = nullptr; |
0 |
| 8759 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) |
0 |
8759 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) |
0 |
| 8760 |
return SDValue(E, 0); |
0 |
8760 |
return SDValue(E, 0); |
0 |
| 8761 |
|
--- |
8761 |
|
--- |
| 8762 |
auto *N = newSDNode( |
0 |
8762 |
auto *N = newSDNode( |
0 |
| 8763 |
dl.getIROrder(), dl.getDebugLoc(), VTs, AM, ST->isTruncatingStore(), |
0 |
8763 |
dl.getIROrder(), dl.getDebugLoc(), VTs, AM, ST->isTruncatingStore(), |
0 |
| 8764 |
ST->isCompressingStore(), ST->getMemoryVT(), ST->getMemOperand()); |
0 |
8764 |
ST->isCompressingStore(), ST->getMemoryVT(), ST->getMemOperand()); |
0 |
| 8765 |
createOperands(N, Ops); |
0 |
8765 |
createOperands(N, Ops); |
0 |
| 8766 |
|
--- |
8766 |
|
--- |
| 8767 |
CSEMap.InsertNode(N, IP); |
0 |
8767 |
CSEMap.InsertNode(N, IP); |
0 |
| 8768 |
InsertNode(N); |
0 |
8768 |
InsertNode(N); |
0 |
| 8769 |
SDValue V(N, 0); |
0 |
8769 |
SDValue V(N, 0); |
0 |
| 8770 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
8770 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 8771 |
return V; |
0 |
8771 |
return V; |
0 |
| 8772 |
} |
0 |
8772 |
} |
0 |
| 8773 |
|
--- |
8773 |
|
--- |
| 8774 |
SDValue SelectionDAG::getStridedLoadVP( |
0 |
8774 |
SDValue SelectionDAG::getStridedLoadVP( |
0 |
| 8775 |
ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, |
--- |
8775 |
ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, |
--- |
| 8776 |
SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, |
--- |
8776 |
SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, |
--- |
| 8777 |
SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, |
--- |
8777 |
SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, |
--- |
| 8778 |
MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, |
--- |
8778 |
MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, |
--- |
| 8779 |
const MDNode *Ranges, bool IsExpanding) { |
--- |
8779 |
const MDNode *Ranges, bool IsExpanding) { |
--- |
| 8780 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
8780 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
| 8781 |
|
--- |
8781 |
|
--- |
| 8782 |
MMOFlags |= MachineMemOperand::MOLoad; |
0 |
8782 |
MMOFlags |= MachineMemOperand::MOLoad; |
0 |
| 8783 |
assert((MMOFlags & MachineMemOperand::MOStore) == 0); |
0 |
8783 |
assert((MMOFlags & MachineMemOperand::MOStore) == 0); |
0 |
| 8784 |
// If we don't have a PtrInfo, infer the trivial frame index case to simplify |
--- |
8784 |
// If we don't have a PtrInfo, infer the trivial frame index case to simplify |
--- |
| 8785 |
// clients. |
--- |
8785 |
// clients. |
--- |
| 8786 |
if (PtrInfo.V.isNull()) |
0 |
8786 |
if (PtrInfo.V.isNull()) |
0 |
| 8787 |
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset); |
0 |
8787 |
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset); |
0 |
| 8788 |
|
--- |
8788 |
|
--- |
| 8789 |
uint64_t Size = MemoryLocation::UnknownSize; |
0 |
8789 |
uint64_t Size = MemoryLocation::UnknownSize; |
0 |
| 8790 |
MachineFunction &MF = getMachineFunction(); |
0 |
8790 |
MachineFunction &MF = getMachineFunction(); |
0 |
| 8791 |
MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, |
0 |
8791 |
MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, |
0 |
| 8792 |
Alignment, AAInfo, Ranges); |
--- |
8792 |
Alignment, AAInfo, Ranges); |
--- |
| 8793 |
return getStridedLoadVP(AM, ExtType, VT, DL, Chain, Ptr, Offset, Stride, Mask, |
0 |
8793 |
return getStridedLoadVP(AM, ExtType, VT, DL, Chain, Ptr, Offset, Stride, Mask, |
0 |
| 8794 |
EVL, MemVT, MMO, IsExpanding); |
0 |
8794 |
EVL, MemVT, MMO, IsExpanding); |
0 |
| 8795 |
} |
--- |
8795 |
} |
--- |
| 8796 |
|
--- |
8796 |
|
--- |
| 8797 |
SDValue SelectionDAG::getStridedLoadVP( |
0 |
8797 |
SDValue SelectionDAG::getStridedLoadVP( |
0 |
| 8798 |
ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, |
--- |
8798 |
ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, |
--- |
| 8799 |
SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, |
--- |
8799 |
SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, |
--- |
| 8800 |
SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) { |
--- |
8800 |
SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) { |
--- |
| 8801 |
bool Indexed = AM != ISD::UNINDEXED; |
0 |
8801 |
bool Indexed = AM != ISD::UNINDEXED; |
0 |
| 8802 |
assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!"); |
0 |
8802 |
assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!"); |
0 |
| 8803 |
|
--- |
8803 |
|
--- |
| 8804 |
SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL}; |
0 |
8804 |
SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL}; |
0 |
| 8805 |
SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other) |
0 |
8805 |
SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other) |
0 |
| 8806 |
: getVTList(VT, MVT::Other); |
0 |
8806 |
: getVTList(VT, MVT::Other); |
0 |
| 8807 |
FoldingSetNodeID ID; |
0 |
8807 |
FoldingSetNodeID ID; |
0 |
| 8808 |
AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTs, Ops); |
0 |
8808 |
AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTs, Ops); |
0 |
| 8809 |
ID.AddInteger(VT.getRawBits()); |
0 |
8809 |
ID.AddInteger(VT.getRawBits()); |
0 |
| 8810 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
8810 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
| 8811 |
DL.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO)); |
--- |
8811 |
DL.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO)); |
--- |
| 8812 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
8812 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
| 8813 |
|
--- |
8813 |
|
--- |
| 8814 |
void *IP = nullptr; |
0 |
8814 |
void *IP = nullptr; |
0 |
| 8815 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { |
0 |
8815 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { |
0 |
| 8816 |
cast(E)->refineAlignment(MMO); |
0 |
8816 |
cast(E)->refineAlignment(MMO); |
0 |
| 8817 |
return SDValue(E, 0); |
0 |
8817 |
return SDValue(E, 0); |
0 |
| 8818 |
} |
--- |
8818 |
} |
--- |
| 8819 |
|
--- |
8819 |
|
--- |
| 8820 |
auto *N = |
--- |
8820 |
auto *N = |
--- |
| 8821 |
newSDNode(DL.getIROrder(), DL.getDebugLoc(), VTs, AM, |
0 |
8821 |
newSDNode(DL.getIROrder(), DL.getDebugLoc(), VTs, AM, |
0 |
| 8822 |
ExtType, IsExpanding, MemVT, MMO); |
--- |
8822 |
ExtType, IsExpanding, MemVT, MMO); |
--- |
| 8823 |
createOperands(N, Ops); |
0 |
8823 |
createOperands(N, Ops); |
0 |
| 8824 |
CSEMap.InsertNode(N, IP); |
0 |
8824 |
CSEMap.InsertNode(N, IP); |
0 |
| 8825 |
InsertNode(N); |
0 |
8825 |
InsertNode(N); |
0 |
| 8826 |
SDValue V(N, 0); |
0 |
8826 |
SDValue V(N, 0); |
0 |
| 8827 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
8827 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 8828 |
return V; |
0 |
8828 |
return V; |
0 |
| 8829 |
} |
0 |
8829 |
} |
0 |
| 8830 |
|
--- |
8830 |
|
--- |
| 8831 |
SDValue SelectionDAG::getStridedLoadVP( |
0 |
8831 |
SDValue SelectionDAG::getStridedLoadVP( |
0 |
| 8832 |
EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Stride, |
--- |
8832 |
EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Stride, |
--- |
| 8833 |
SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, MaybeAlign Alignment, |
--- |
8833 |
SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, MaybeAlign Alignment, |
--- |
| 8834 |
MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, |
--- |
8834 |
MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, |
--- |
| 8835 |
const MDNode *Ranges, bool IsExpanding) { |
--- |
8835 |
const MDNode *Ranges, bool IsExpanding) { |
--- |
| 8836 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
8836 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
| 8837 |
return getStridedLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, DL, Chain, Ptr, |
0 |
8837 |
return getStridedLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, DL, Chain, Ptr, |
0 |
| 8838 |
Undef, Stride, Mask, EVL, PtrInfo, VT, Alignment, |
--- |
8838 |
Undef, Stride, Mask, EVL, PtrInfo, VT, Alignment, |
--- |
| 8839 |
MMOFlags, AAInfo, Ranges, IsExpanding); |
0 |
8839 |
MMOFlags, AAInfo, Ranges, IsExpanding); |
0 |
| 8840 |
} |
--- |
8840 |
} |
--- |
| 8841 |
|
--- |
8841 |
|
--- |
| 8842 |
SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain, |
0 |
8842 |
SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain, |
0 |
| 8843 |
SDValue Ptr, SDValue Stride, |
--- |
8843 |
SDValue Ptr, SDValue Stride, |
--- |
| 8844 |
SDValue Mask, SDValue EVL, |
--- |
8844 |
SDValue Mask, SDValue EVL, |
--- |
| 8845 |
MachineMemOperand *MMO, |
--- |
8845 |
MachineMemOperand *MMO, |
--- |
| 8846 |
bool IsExpanding) { |
--- |
8846 |
bool IsExpanding) { |
--- |
| 8847 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
8847 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
| 8848 |
return getStridedLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, DL, Chain, Ptr, |
0 |
8848 |
return getStridedLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, DL, Chain, Ptr, |
0 |
| 8849 |
Undef, Stride, Mask, EVL, VT, MMO, IsExpanding); |
0 |
8849 |
Undef, Stride, Mask, EVL, VT, MMO, IsExpanding); |
0 |
| 8850 |
} |
--- |
8850 |
} |
--- |
| 8851 |
|
--- |
8851 |
|
--- |
| 8852 |
SDValue SelectionDAG::getExtStridedLoadVP( |
0 |
8852 |
SDValue SelectionDAG::getExtStridedLoadVP( |
0 |
| 8853 |
ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, |
--- |
8853 |
ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, |
--- |
| 8854 |
SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, |
--- |
8854 |
SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, |
--- |
| 8855 |
MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment, |
--- |
8855 |
MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment, |
--- |
| 8856 |
MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, |
--- |
8856 |
MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, |
--- |
| 8857 |
bool IsExpanding) { |
--- |
8857 |
bool IsExpanding) { |
--- |
| 8858 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
8858 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
| 8859 |
return getStridedLoadVP(ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Undef, |
0 |
8859 |
return getStridedLoadVP(ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Undef, |
0 |
| 8860 |
Stride, Mask, EVL, PtrInfo, MemVT, Alignment, |
--- |
8860 |
Stride, Mask, EVL, PtrInfo, MemVT, Alignment, |
--- |
| 8861 |
MMOFlags, AAInfo, nullptr, IsExpanding); |
0 |
8861 |
MMOFlags, AAInfo, nullptr, IsExpanding); |
0 |
| 8862 |
} |
--- |
8862 |
} |
--- |
| 8863 |
|
--- |
8863 |
|
--- |
| 8864 |
SDValue SelectionDAG::getExtStridedLoadVP( |
0 |
8864 |
SDValue SelectionDAG::getExtStridedLoadVP( |
0 |
| 8865 |
ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, |
--- |
8865 |
ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, |
--- |
| 8866 |
SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, |
--- |
8866 |
SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, |
--- |
| 8867 |
MachineMemOperand *MMO, bool IsExpanding) { |
--- |
8867 |
MachineMemOperand *MMO, bool IsExpanding) { |
--- |
| 8868 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
8868 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
| 8869 |
return getStridedLoadVP(ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Undef, |
0 |
8869 |
return getStridedLoadVP(ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Undef, |
0 |
| 8870 |
Stride, Mask, EVL, MemVT, MMO, IsExpanding); |
0 |
8870 |
Stride, Mask, EVL, MemVT, MMO, IsExpanding); |
0 |
| 8871 |
} |
--- |
8871 |
} |
--- |
| 8872 |
|
--- |
8872 |
|
--- |
| 8873 |
SDValue SelectionDAG::getIndexedStridedLoadVP(SDValue OrigLoad, const SDLoc &DL, |
0 |
8873 |
SDValue SelectionDAG::getIndexedStridedLoadVP(SDValue OrigLoad, const SDLoc &DL, |
0 |
| 8874 |
SDValue Base, SDValue Offset, |
--- |
8874 |
SDValue Base, SDValue Offset, |
--- |
| 8875 |
ISD::MemIndexedMode AM) { |
--- |
8875 |
ISD::MemIndexedMode AM) { |
--- |
| 8876 |
auto *SLD = cast(OrigLoad); |
0 |
8876 |
auto *SLD = cast(OrigLoad); |
0 |
| 8877 |
assert(SLD->getOffset().isUndef() && |
0 |
8877 |
assert(SLD->getOffset().isUndef() && |
0 |
| 8878 |
"Strided load is already a indexed load!"); |
--- |
8878 |
"Strided load is already a indexed load!"); |
--- |
| 8879 |
// Don't propagate the invariant or dereferenceable flags. |
--- |
8879 |
// Don't propagate the invariant or dereferenceable flags. |
--- |
| 8880 |
auto MMOFlags = |
--- |
8880 |
auto MMOFlags = |
--- |
| 8881 |
SLD->getMemOperand()->getFlags() & |
0 |
8881 |
SLD->getMemOperand()->getFlags() & |
0 |
| 8882 |
~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); |
0 |
8882 |
~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); |
0 |
| 8883 |
return getStridedLoadVP( |
0 |
8883 |
return getStridedLoadVP( |
0 |
| 8884 |
AM, SLD->getExtensionType(), OrigLoad.getValueType(), DL, SLD->getChain(), |
0 |
8884 |
AM, SLD->getExtensionType(), OrigLoad.getValueType(), DL, SLD->getChain(), |
0 |
| 8885 |
Base, Offset, SLD->getStride(), SLD->getMask(), SLD->getVectorLength(), |
0 |
8885 |
Base, Offset, SLD->getStride(), SLD->getMask(), SLD->getVectorLength(), |
0 |
| 8886 |
SLD->getPointerInfo(), SLD->getMemoryVT(), SLD->getAlign(), MMOFlags, |
0 |
8886 |
SLD->getPointerInfo(), SLD->getMemoryVT(), SLD->getAlign(), MMOFlags, |
0 |
| 8887 |
SLD->getAAInfo(), nullptr, SLD->isExpandingLoad()); |
0 |
8887 |
SLD->getAAInfo(), nullptr, SLD->isExpandingLoad()); |
0 |
| 8888 |
} |
--- |
8888 |
} |
--- |
| 8889 |
|
--- |
8889 |
|
--- |
| 8890 |
SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL, |
0 |
8890 |
SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL, |
0 |
| 8891 |
SDValue Val, SDValue Ptr, |
--- |
8891 |
SDValue Val, SDValue Ptr, |
--- |
| 8892 |
SDValue Offset, SDValue Stride, |
--- |
8892 |
SDValue Offset, SDValue Stride, |
--- |
| 8893 |
SDValue Mask, SDValue EVL, EVT MemVT, |
--- |
8893 |
SDValue Mask, SDValue EVL, EVT MemVT, |
--- |
| 8894 |
MachineMemOperand *MMO, |
--- |
8894 |
MachineMemOperand *MMO, |
--- |
| 8895 |
ISD::MemIndexedMode AM, |
--- |
8895 |
ISD::MemIndexedMode AM, |
--- |
| 8896 |
bool IsTruncating, bool IsCompressing) { |
--- |
8896 |
bool IsTruncating, bool IsCompressing) { |
--- |
| 8897 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
8897 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
| 8898 |
bool Indexed = AM != ISD::UNINDEXED; |
0 |
8898 |
bool Indexed = AM != ISD::UNINDEXED; |
0 |
| 8899 |
assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!"); |
0 |
8899 |
assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!"); |
0 |
| 8900 |
SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other) |
0 |
8900 |
SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other) |
0 |
| 8901 |
: getVTList(MVT::Other); |
0 |
8901 |
: getVTList(MVT::Other); |
0 |
| 8902 |
SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL}; |
0 |
8902 |
SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL}; |
0 |
| 8903 |
FoldingSetNodeID ID; |
0 |
8903 |
FoldingSetNodeID ID; |
0 |
| 8904 |
AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops); |
0 |
8904 |
AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops); |
0 |
| 8905 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
8905 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
| 8906 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
8906 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
| 8907 |
DL.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO)); |
--- |
8907 |
DL.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO)); |
--- |
| 8908 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
8908 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
| 8909 |
void *IP = nullptr; |
0 |
8909 |
void *IP = nullptr; |
0 |
| 8910 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { |
0 |
8910 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { |
0 |
| 8911 |
cast(E)->refineAlignment(MMO); |
0 |
8911 |
cast(E)->refineAlignment(MMO); |
0 |
| 8912 |
return SDValue(E, 0); |
0 |
8912 |
return SDValue(E, 0); |
0 |
| 8913 |
} |
--- |
8913 |
} |
--- |
| 8914 |
auto *N = newSDNode(DL.getIROrder(), DL.getDebugLoc(), |
0 |
8914 |
auto *N = newSDNode(DL.getIROrder(), DL.getDebugLoc(), |
0 |
| 8915 |
VTs, AM, IsTruncating, |
--- |
8915 |
VTs, AM, IsTruncating, |
--- |
| 8916 |
IsCompressing, MemVT, MMO); |
--- |
8916 |
IsCompressing, MemVT, MMO); |
--- |
| 8917 |
createOperands(N, Ops); |
0 |
8917 |
createOperands(N, Ops); |
0 |
| 8918 |
|
--- |
8918 |
|
--- |
| 8919 |
CSEMap.InsertNode(N, IP); |
0 |
8919 |
CSEMap.InsertNode(N, IP); |
0 |
| 8920 |
InsertNode(N); |
0 |
8920 |
InsertNode(N); |
0 |
| 8921 |
SDValue V(N, 0); |
0 |
8921 |
SDValue V(N, 0); |
0 |
| 8922 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
8922 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 8923 |
return V; |
0 |
8923 |
return V; |
0 |
| 8924 |
} |
0 |
8924 |
} |
0 |
| 8925 |
|
--- |
8925 |
|
--- |
| 8926 |
SDValue SelectionDAG::getTruncStridedStoreVP( |
0 |
8926 |
SDValue SelectionDAG::getTruncStridedStoreVP( |
0 |
| 8927 |
SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Stride, |
--- |
8927 |
SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Stride, |
--- |
| 8928 |
SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT, |
--- |
8928 |
SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT, |
--- |
| 8929 |
Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, |
--- |
8929 |
Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, |
--- |
| 8930 |
bool IsCompressing) { |
--- |
8930 |
bool IsCompressing) { |
--- |
| 8931 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
8931 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
| 8932 |
|
--- |
8932 |
|
--- |
| 8933 |
MMOFlags |= MachineMemOperand::MOStore; |
0 |
8933 |
MMOFlags |= MachineMemOperand::MOStore; |
0 |
| 8934 |
assert((MMOFlags & MachineMemOperand::MOLoad) == 0); |
0 |
8934 |
assert((MMOFlags & MachineMemOperand::MOLoad) == 0); |
0 |
| 8935 |
|
--- |
8935 |
|
--- |
| 8936 |
if (PtrInfo.V.isNull()) |
0 |
8936 |
if (PtrInfo.V.isNull()) |
0 |
| 8937 |
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr); |
0 |
8937 |
PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr); |
0 |
| 8938 |
|
--- |
8938 |
|
--- |
| 8939 |
MachineFunction &MF = getMachineFunction(); |
0 |
8939 |
MachineFunction &MF = getMachineFunction(); |
0 |
| 8940 |
MachineMemOperand *MMO = MF.getMachineMemOperand( |
0 |
8940 |
MachineMemOperand *MMO = MF.getMachineMemOperand( |
0 |
| 8941 |
PtrInfo, MMOFlags, MemoryLocation::UnknownSize, Alignment, AAInfo); |
--- |
8941 |
PtrInfo, MMOFlags, MemoryLocation::UnknownSize, Alignment, AAInfo); |
--- |
| 8942 |
return getTruncStridedStoreVP(Chain, DL, Val, Ptr, Stride, Mask, EVL, SVT, |
0 |
8942 |
return getTruncStridedStoreVP(Chain, DL, Val, Ptr, Stride, Mask, EVL, SVT, |
0 |
| 8943 |
MMO, IsCompressing); |
0 |
8943 |
MMO, IsCompressing); |
0 |
| 8944 |
} |
--- |
8944 |
} |
--- |
| 8945 |
|
--- |
8945 |
|
--- |
| 8946 |
SDValue SelectionDAG::getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, |
0 |
8946 |
SDValue SelectionDAG::getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, |
0 |
| 8947 |
SDValue Val, SDValue Ptr, |
--- |
8947 |
SDValue Val, SDValue Ptr, |
--- |
| 8948 |
SDValue Stride, SDValue Mask, |
--- |
8948 |
SDValue Stride, SDValue Mask, |
--- |
| 8949 |
SDValue EVL, EVT SVT, |
--- |
8949 |
SDValue EVL, EVT SVT, |
--- |
| 8950 |
MachineMemOperand *MMO, |
--- |
8950 |
MachineMemOperand *MMO, |
--- |
| 8951 |
bool IsCompressing) { |
--- |
8951 |
bool IsCompressing) { |
--- |
| 8952 |
EVT VT = Val.getValueType(); |
0 |
8952 |
EVT VT = Val.getValueType(); |
0 |
| 8953 |
|
--- |
8953 |
|
--- |
| 8954 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
8954 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
| 8955 |
if (VT == SVT) |
0 |
8955 |
if (VT == SVT) |
0 |
| 8956 |
return getStridedStoreVP(Chain, DL, Val, Ptr, getUNDEF(Ptr.getValueType()), |
0 |
8956 |
return getStridedStoreVP(Chain, DL, Val, Ptr, getUNDEF(Ptr.getValueType()), |
0 |
| 8957 |
Stride, Mask, EVL, VT, MMO, ISD::UNINDEXED, |
--- |
8957 |
Stride, Mask, EVL, VT, MMO, ISD::UNINDEXED, |
--- |
| 8958 |
/*IsTruncating*/ false, IsCompressing); |
0 |
8958 |
/*IsTruncating*/ false, IsCompressing); |
0 |
| 8959 |
|
--- |
8959 |
|
--- |
| 8960 |
assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && |
0 |
8960 |
assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && |
0 |
| 8961 |
"Should only be a truncating store, not extending!"); |
--- |
8961 |
"Should only be a truncating store, not extending!"); |
--- |
| 8962 |
assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!"); |
0 |
8962 |
assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!"); |
0 |
| 8963 |
assert(VT.isVector() == SVT.isVector() && |
0 |
8963 |
assert(VT.isVector() == SVT.isVector() && |
0 |
| 8964 |
"Cannot use trunc store to convert to or from a vector!"); |
--- |
8964 |
"Cannot use trunc store to convert to or from a vector!"); |
--- |
| 8965 |
assert((!VT.isVector() || |
0 |
8965 |
assert((!VT.isVector() || |
0 |
| 8966 |
VT.getVectorElementCount() == SVT.getVectorElementCount()) && |
--- |
8966 |
VT.getVectorElementCount() == SVT.getVectorElementCount()) && |
--- |
| 8967 |
"Cannot use trunc store to change the number of vector elements!"); |
--- |
8967 |
"Cannot use trunc store to change the number of vector elements!"); |
--- |
| 8968 |
|
--- |
8968 |
|
--- |
| 8969 |
SDVTList VTs = getVTList(MVT::Other); |
0 |
8969 |
SDVTList VTs = getVTList(MVT::Other); |
0 |
| 8970 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
8970 |
SDValue Undef = getUNDEF(Ptr.getValueType()); |
0 |
| 8971 |
SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL}; |
0 |
8971 |
SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL}; |
0 |
| 8972 |
FoldingSetNodeID ID; |
0 |
8972 |
FoldingSetNodeID ID; |
0 |
| 8973 |
AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops); |
0 |
8973 |
AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops); |
0 |
| 8974 |
ID.AddInteger(SVT.getRawBits()); |
0 |
8974 |
ID.AddInteger(SVT.getRawBits()); |
0 |
| 8975 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
8975 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
| 8976 |
DL.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO)); |
0 |
8976 |
DL.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO)); |
0 |
| 8977 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
8977 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
| 8978 |
void *IP = nullptr; |
0 |
8978 |
void *IP = nullptr; |
0 |
| 8979 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { |
0 |
8979 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { |
0 |
| 8980 |
cast(E)->refineAlignment(MMO); |
0 |
8980 |
cast(E)->refineAlignment(MMO); |
0 |
| 8981 |
return SDValue(E, 0); |
0 |
8981 |
return SDValue(E, 0); |
0 |
| 8982 |
} |
--- |
8982 |
} |
--- |
| 8983 |
auto *N = newSDNode(DL.getIROrder(), DL.getDebugLoc(), |
0 |
8983 |
auto *N = newSDNode(DL.getIROrder(), DL.getDebugLoc(), |
0 |
| 8984 |
VTs, ISD::UNINDEXED, true, |
0 |
8984 |
VTs, ISD::UNINDEXED, true, |
0 |
| 8985 |
IsCompressing, SVT, MMO); |
--- |
8985 |
IsCompressing, SVT, MMO); |
--- |
| 8986 |
createOperands(N, Ops); |
0 |
8986 |
createOperands(N, Ops); |
0 |
| 8987 |
|
--- |
8987 |
|
--- |
| 8988 |
CSEMap.InsertNode(N, IP); |
0 |
8988 |
CSEMap.InsertNode(N, IP); |
0 |
| 8989 |
InsertNode(N); |
0 |
8989 |
InsertNode(N); |
0 |
| 8990 |
SDValue V(N, 0); |
0 |
8990 |
SDValue V(N, 0); |
0 |
| 8991 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
8991 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 8992 |
return V; |
0 |
8992 |
return V; |
0 |
| 8993 |
} |
0 |
8993 |
} |
0 |
| 8994 |
|
--- |
8994 |
|
--- |
| 8995 |
SDValue SelectionDAG::getIndexedStridedStoreVP(SDValue OrigStore, |
0 |
8995 |
SDValue SelectionDAG::getIndexedStridedStoreVP(SDValue OrigStore, |
0 |
| 8996 |
const SDLoc &DL, SDValue Base, |
--- |
8996 |
const SDLoc &DL, SDValue Base, |
--- |
| 8997 |
SDValue Offset, |
--- |
8997 |
SDValue Offset, |
--- |
| 8998 |
ISD::MemIndexedMode AM) { |
--- |
8998 |
ISD::MemIndexedMode AM) { |
--- |
| 8999 |
auto *SST = cast(OrigStore); |
0 |
8999 |
auto *SST = cast(OrigStore); |
0 |
| 9000 |
assert(SST->getOffset().isUndef() && |
0 |
9000 |
assert(SST->getOffset().isUndef() && |
0 |
| 9001 |
"Strided store is already an indexed store!"); |
--- |
9001 |
"Strided store is already an indexed store!"); |
--- |
| 9002 |
SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); |
0 |
9002 |
SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); |
0 |
| 9003 |
SDValue Ops[] = { |
--- |
9003 |
SDValue Ops[] = { |
--- |
| 9004 |
SST->getChain(), SST->getValue(), Base, Offset, SST->getStride(), |
0 |
9004 |
SST->getChain(), SST->getValue(), Base, Offset, SST->getStride(), |
0 |
| 9005 |
SST->getMask(), SST->getVectorLength()}; |
0 |
9005 |
SST->getMask(), SST->getVectorLength()}; |
0 |
| 9006 |
FoldingSetNodeID ID; |
0 |
9006 |
FoldingSetNodeID ID; |
0 |
| 9007 |
AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops); |
0 |
9007 |
AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops); |
0 |
| 9008 |
ID.AddInteger(SST->getMemoryVT().getRawBits()); |
0 |
9008 |
ID.AddInteger(SST->getMemoryVT().getRawBits()); |
0 |
| 9009 |
ID.AddInteger(SST->getRawSubclassData()); |
0 |
9009 |
ID.AddInteger(SST->getRawSubclassData()); |
0 |
| 9010 |
ID.AddInteger(SST->getPointerInfo().getAddrSpace()); |
0 |
9010 |
ID.AddInteger(SST->getPointerInfo().getAddrSpace()); |
0 |
| 9011 |
void *IP = nullptr; |
0 |
9011 |
void *IP = nullptr; |
0 |
| 9012 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) |
0 |
9012 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) |
0 |
| 9013 |
return SDValue(E, 0); |
0 |
9013 |
return SDValue(E, 0); |
0 |
| 9014 |
|
--- |
9014 |
|
--- |
| 9015 |
auto *N = newSDNode( |
0 |
9015 |
auto *N = newSDNode( |
0 |
| 9016 |
DL.getIROrder(), DL.getDebugLoc(), VTs, AM, SST->isTruncatingStore(), |
0 |
9016 |
DL.getIROrder(), DL.getDebugLoc(), VTs, AM, SST->isTruncatingStore(), |
0 |
| 9017 |
SST->isCompressingStore(), SST->getMemoryVT(), SST->getMemOperand()); |
0 |
9017 |
SST->isCompressingStore(), SST->getMemoryVT(), SST->getMemOperand()); |
0 |
| 9018 |
createOperands(N, Ops); |
0 |
9018 |
createOperands(N, Ops); |
0 |
| 9019 |
|
--- |
9019 |
|
--- |
| 9020 |
CSEMap.InsertNode(N, IP); |
0 |
9020 |
CSEMap.InsertNode(N, IP); |
0 |
| 9021 |
InsertNode(N); |
0 |
9021 |
InsertNode(N); |
0 |
| 9022 |
SDValue V(N, 0); |
0 |
9022 |
SDValue V(N, 0); |
0 |
| 9023 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
9023 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 9024 |
return V; |
0 |
9024 |
return V; |
0 |
| 9025 |
} |
0 |
9025 |
} |
0 |
| 9026 |
|
--- |
9026 |
|
--- |
| 9027 |
SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, |
0 |
9027 |
SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, |
0 |
| 9028 |
ArrayRef Ops, MachineMemOperand *MMO, |
--- |
9028 |
ArrayRef Ops, MachineMemOperand *MMO, |
--- |
| 9029 |
ISD::MemIndexType IndexType) { |
--- |
9029 |
ISD::MemIndexType IndexType) { |
--- |
| 9030 |
assert(Ops.size() == 6 && "Incompatible number of operands"); |
0 |
9030 |
assert(Ops.size() == 6 && "Incompatible number of operands"); |
0 |
| 9031 |
|
--- |
9031 |
|
--- |
| 9032 |
FoldingSetNodeID ID; |
0 |
9032 |
FoldingSetNodeID ID; |
0 |
| 9033 |
AddNodeIDNode(ID, ISD::VP_GATHER, VTs, Ops); |
0 |
9033 |
AddNodeIDNode(ID, ISD::VP_GATHER, VTs, Ops); |
0 |
| 9034 |
ID.AddInteger(VT.getRawBits()); |
0 |
9034 |
ID.AddInteger(VT.getRawBits()); |
0 |
| 9035 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
9035 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
| 9036 |
dl.getIROrder(), VTs, VT, MMO, IndexType)); |
--- |
9036 |
dl.getIROrder(), VTs, VT, MMO, IndexType)); |
--- |
| 9037 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
9037 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
| 9038 |
ID.AddInteger(MMO->getFlags()); |
0 |
9038 |
ID.AddInteger(MMO->getFlags()); |
0 |
| 9039 |
void *IP = nullptr; |
0 |
9039 |
void *IP = nullptr; |
0 |
| 9040 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
9040 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
| 9041 |
cast(E)->refineAlignment(MMO); |
0 |
9041 |
cast(E)->refineAlignment(MMO); |
0 |
| 9042 |
return SDValue(E, 0); |
0 |
9042 |
return SDValue(E, 0); |
0 |
| 9043 |
} |
--- |
9043 |
} |
--- |
| 9044 |
|
--- |
9044 |
|
--- |
| 9045 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, |
0 |
9045 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, |
0 |
| 9046 |
VT, MMO, IndexType); |
--- |
9046 |
VT, MMO, IndexType); |
--- |
| 9047 |
createOperands(N, Ops); |
0 |
9047 |
createOperands(N, Ops); |
0 |
| 9048 |
|
--- |
9048 |
|
--- |
| 9049 |
assert(N->getMask().getValueType().getVectorElementCount() == |
0 |
9049 |
assert(N->getMask().getValueType().getVectorElementCount() == |
0 |
| 9050 |
N->getValueType(0).getVectorElementCount() && |
--- |
9050 |
N->getValueType(0).getVectorElementCount() && |
--- |
| 9051 |
"Vector width mismatch between mask and data"); |
--- |
9051 |
"Vector width mismatch between mask and data"); |
--- |
| 9052 |
assert(N->getIndex().getValueType().getVectorElementCount().isScalable() == |
0 |
9052 |
assert(N->getIndex().getValueType().getVectorElementCount().isScalable() == |
0 |
| 9053 |
N->getValueType(0).getVectorElementCount().isScalable() && |
--- |
9053 |
N->getValueType(0).getVectorElementCount().isScalable() && |
--- |
| 9054 |
"Scalable flags of index and data do not match"); |
--- |
9054 |
"Scalable flags of index and data do not match"); |
--- |
| 9055 |
assert(ElementCount::isKnownGE( |
0 |
9055 |
assert(ElementCount::isKnownGE( |
0 |
| 9056 |
N->getIndex().getValueType().getVectorElementCount(), |
--- |
9056 |
N->getIndex().getValueType().getVectorElementCount(), |
--- |
| 9057 |
N->getValueType(0).getVectorElementCount()) && |
--- |
9057 |
N->getValueType(0).getVectorElementCount()) && |
--- |
| 9058 |
"Vector width mismatch between index and data"); |
--- |
9058 |
"Vector width mismatch between index and data"); |
--- |
| 9059 |
assert(isa(N->getScale()) && |
0 |
9059 |
assert(isa(N->getScale()) && |
0 |
| 9060 |
cast(N->getScale())->getAPIntValue().isPowerOf2() && |
--- |
9060 |
cast(N->getScale())->getAPIntValue().isPowerOf2() && |
--- |
| 9061 |
"Scale should be a constant power of 2"); |
--- |
9061 |
"Scale should be a constant power of 2"); |
--- |
| 9062 |
|
--- |
9062 |
|
--- |
| 9063 |
CSEMap.InsertNode(N, IP); |
0 |
9063 |
CSEMap.InsertNode(N, IP); |
0 |
| 9064 |
InsertNode(N); |
0 |
9064 |
InsertNode(N); |
0 |
| 9065 |
SDValue V(N, 0); |
0 |
9065 |
SDValue V(N, 0); |
0 |
| 9066 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
9066 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 9067 |
return V; |
0 |
9067 |
return V; |
0 |
| 9068 |
} |
0 |
9068 |
} |
0 |
| 9069 |
|
--- |
9069 |
|
--- |
| 9070 |
SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, |
0 |
9070 |
SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, |
0 |
| 9071 |
ArrayRef Ops, |
--- |
9071 |
ArrayRef Ops, |
--- |
| 9072 |
MachineMemOperand *MMO, |
--- |
9072 |
MachineMemOperand *MMO, |
--- |
| 9073 |
ISD::MemIndexType IndexType) { |
--- |
9073 |
ISD::MemIndexType IndexType) { |
--- |
| 9074 |
assert(Ops.size() == 7 && "Incompatible number of operands"); |
0 |
9074 |
assert(Ops.size() == 7 && "Incompatible number of operands"); |
0 |
| 9075 |
|
--- |
9075 |
|
--- |
| 9076 |
FoldingSetNodeID ID; |
0 |
9076 |
FoldingSetNodeID ID; |
0 |
| 9077 |
AddNodeIDNode(ID, ISD::VP_SCATTER, VTs, Ops); |
0 |
9077 |
AddNodeIDNode(ID, ISD::VP_SCATTER, VTs, Ops); |
0 |
| 9078 |
ID.AddInteger(VT.getRawBits()); |
0 |
9078 |
ID.AddInteger(VT.getRawBits()); |
0 |
| 9079 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
9079 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
| 9080 |
dl.getIROrder(), VTs, VT, MMO, IndexType)); |
--- |
9080 |
dl.getIROrder(), VTs, VT, MMO, IndexType)); |
--- |
| 9081 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
9081 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
| 9082 |
ID.AddInteger(MMO->getFlags()); |
0 |
9082 |
ID.AddInteger(MMO->getFlags()); |
0 |
| 9083 |
void *IP = nullptr; |
0 |
9083 |
void *IP = nullptr; |
0 |
| 9084 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
9084 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
| 9085 |
cast(E)->refineAlignment(MMO); |
0 |
9085 |
cast(E)->refineAlignment(MMO); |
0 |
| 9086 |
return SDValue(E, 0); |
0 |
9086 |
return SDValue(E, 0); |
0 |
| 9087 |
} |
--- |
9087 |
} |
--- |
| 9088 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, |
0 |
9088 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, |
0 |
| 9089 |
VT, MMO, IndexType); |
--- |
9089 |
VT, MMO, IndexType); |
--- |
| 9090 |
createOperands(N, Ops); |
0 |
9090 |
createOperands(N, Ops); |
0 |
| 9091 |
|
--- |
9091 |
|
--- |
| 9092 |
assert(N->getMask().getValueType().getVectorElementCount() == |
0 |
9092 |
assert(N->getMask().getValueType().getVectorElementCount() == |
0 |
| 9093 |
N->getValue().getValueType().getVectorElementCount() && |
--- |
9093 |
N->getValue().getValueType().getVectorElementCount() && |
--- |
| 9094 |
"Vector width mismatch between mask and data"); |
--- |
9094 |
"Vector width mismatch between mask and data"); |
--- |
| 9095 |
assert( |
0 |
9095 |
assert( |
0 |
| 9096 |
N->getIndex().getValueType().getVectorElementCount().isScalable() == |
--- |
9096 |
N->getIndex().getValueType().getVectorElementCount().isScalable() == |
--- |
| 9097 |
N->getValue().getValueType().getVectorElementCount().isScalable() && |
--- |
9097 |
N->getValue().getValueType().getVectorElementCount().isScalable() && |
--- |
| 9098 |
"Scalable flags of index and data do not match"); |
--- |
9098 |
"Scalable flags of index and data do not match"); |
--- |
| 9099 |
assert(ElementCount::isKnownGE( |
0 |
9099 |
assert(ElementCount::isKnownGE( |
0 |
| 9100 |
N->getIndex().getValueType().getVectorElementCount(), |
--- |
9100 |
N->getIndex().getValueType().getVectorElementCount(), |
--- |
| 9101 |
N->getValue().getValueType().getVectorElementCount()) && |
--- |
9101 |
N->getValue().getValueType().getVectorElementCount()) && |
--- |
| 9102 |
"Vector width mismatch between index and data"); |
--- |
9102 |
"Vector width mismatch between index and data"); |
--- |
| 9103 |
assert(isa(N->getScale()) && |
0 |
9103 |
assert(isa(N->getScale()) && |
0 |
| 9104 |
cast(N->getScale())->getAPIntValue().isPowerOf2() && |
--- |
9104 |
cast(N->getScale())->getAPIntValue().isPowerOf2() && |
--- |
| 9105 |
"Scale should be a constant power of 2"); |
--- |
9105 |
"Scale should be a constant power of 2"); |
--- |
| 9106 |
|
--- |
9106 |
|
--- |
| 9107 |
CSEMap.InsertNode(N, IP); |
0 |
9107 |
CSEMap.InsertNode(N, IP); |
0 |
| 9108 |
InsertNode(N); |
0 |
9108 |
InsertNode(N); |
0 |
| 9109 |
SDValue V(N, 0); |
0 |
9109 |
SDValue V(N, 0); |
0 |
| 9110 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
9110 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 9111 |
return V; |
0 |
9111 |
return V; |
0 |
| 9112 |
} |
0 |
9112 |
} |
0 |
| 9113 |
|
--- |
9113 |
|
--- |
| 9114 |
SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, |
0 |
9114 |
SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, |
0 |
| 9115 |
SDValue Base, SDValue Offset, SDValue Mask, |
--- |
9115 |
SDValue Base, SDValue Offset, SDValue Mask, |
--- |
| 9116 |
SDValue PassThru, EVT MemVT, |
--- |
9116 |
SDValue PassThru, EVT MemVT, |
--- |
| 9117 |
MachineMemOperand *MMO, |
--- |
9117 |
MachineMemOperand *MMO, |
--- |
| 9118 |
ISD::MemIndexedMode AM, |
--- |
9118 |
ISD::MemIndexedMode AM, |
--- |
| 9119 |
ISD::LoadExtType ExtTy, bool isExpanding) { |
--- |
9119 |
ISD::LoadExtType ExtTy, bool isExpanding) { |
--- |
| 9120 |
bool Indexed = AM != ISD::UNINDEXED; |
0 |
9120 |
bool Indexed = AM != ISD::UNINDEXED; |
0 |
| 9121 |
assert((Indexed || Offset.isUndef()) && |
0 |
9121 |
assert((Indexed || Offset.isUndef()) && |
0 |
| 9122 |
"Unindexed masked load with an offset!"); |
--- |
9122 |
"Unindexed masked load with an offset!"); |
--- |
| 9123 |
SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other) |
0 |
9123 |
SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other) |
0 |
| 9124 |
: getVTList(VT, MVT::Other); |
0 |
9124 |
: getVTList(VT, MVT::Other); |
0 |
| 9125 |
SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru}; |
0 |
9125 |
SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru}; |
0 |
| 9126 |
FoldingSetNodeID ID; |
0 |
9126 |
FoldingSetNodeID ID; |
0 |
| 9127 |
AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops); |
0 |
9127 |
AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops); |
0 |
| 9128 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
9128 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
| 9129 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
9129 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
| 9130 |
dl.getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO)); |
--- |
9130 |
dl.getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO)); |
--- |
| 9131 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
9131 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
| 9132 |
ID.AddInteger(MMO->getFlags()); |
0 |
9132 |
ID.AddInteger(MMO->getFlags()); |
0 |
| 9133 |
void *IP = nullptr; |
0 |
9133 |
void *IP = nullptr; |
0 |
| 9134 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
9134 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
| 9135 |
cast(E)->refineAlignment(MMO); |
0 |
9135 |
cast(E)->refineAlignment(MMO); |
0 |
| 9136 |
return SDValue(E, 0); |
0 |
9136 |
return SDValue(E, 0); |
0 |
| 9137 |
} |
--- |
9137 |
} |
--- |
| 9138 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, |
0 |
9138 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, |
0 |
| 9139 |
AM, ExtTy, isExpanding, MemVT, MMO); |
--- |
9139 |
AM, ExtTy, isExpanding, MemVT, MMO); |
--- |
| 9140 |
createOperands(N, Ops); |
0 |
9140 |
createOperands(N, Ops); |
0 |
| 9141 |
|
--- |
9141 |
|
--- |
| 9142 |
CSEMap.InsertNode(N, IP); |
0 |
9142 |
CSEMap.InsertNode(N, IP); |
0 |
| 9143 |
InsertNode(N); |
0 |
9143 |
InsertNode(N); |
0 |
| 9144 |
SDValue V(N, 0); |
0 |
9144 |
SDValue V(N, 0); |
0 |
| 9145 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
9145 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 9146 |
return V; |
0 |
9146 |
return V; |
0 |
| 9147 |
} |
0 |
9147 |
} |
0 |
| 9148 |
|
--- |
9148 |
|
--- |
| 9149 |
SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, |
0 |
9149 |
SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, |
0 |
| 9150 |
SDValue Base, SDValue Offset, |
--- |
9150 |
SDValue Base, SDValue Offset, |
--- |
| 9151 |
ISD::MemIndexedMode AM) { |
--- |
9151 |
ISD::MemIndexedMode AM) { |
--- |
| 9152 |
MaskedLoadSDNode *LD = cast(OrigLoad); |
0 |
9152 |
MaskedLoadSDNode *LD = cast(OrigLoad); |
0 |
| 9153 |
assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!"); |
0 |
9153 |
assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!"); |
0 |
| 9154 |
return getMaskedLoad(OrigLoad.getValueType(), dl, LD->getChain(), Base, |
0 |
9154 |
return getMaskedLoad(OrigLoad.getValueType(), dl, LD->getChain(), Base, |
0 |
| 9155 |
Offset, LD->getMask(), LD->getPassThru(), |
0 |
9155 |
Offset, LD->getMask(), LD->getPassThru(), |
0 |
| 9156 |
LD->getMemoryVT(), LD->getMemOperand(), AM, |
--- |
9156 |
LD->getMemoryVT(), LD->getMemOperand(), AM, |
--- |
| 9157 |
LD->getExtensionType(), LD->isExpandingLoad()); |
0 |
9157 |
LD->getExtensionType(), LD->isExpandingLoad()); |
0 |
| 9158 |
} |
--- |
9158 |
} |
--- |
| 9159 |
|
--- |
9159 |
|
--- |
| 9160 |
SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl, |
0 |
9160 |
SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl, |
0 |
| 9161 |
SDValue Val, SDValue Base, SDValue Offset, |
--- |
9161 |
SDValue Val, SDValue Base, SDValue Offset, |
--- |
| 9162 |
SDValue Mask, EVT MemVT, |
--- |
9162 |
SDValue Mask, EVT MemVT, |
--- |
| 9163 |
MachineMemOperand *MMO, |
--- |
9163 |
MachineMemOperand *MMO, |
--- |
| 9164 |
ISD::MemIndexedMode AM, bool IsTruncating, |
--- |
9164 |
ISD::MemIndexedMode AM, bool IsTruncating, |
--- |
| 9165 |
bool IsCompressing) { |
--- |
9165 |
bool IsCompressing) { |
--- |
| 9166 |
assert(Chain.getValueType() == MVT::Other && |
0 |
9166 |
assert(Chain.getValueType() == MVT::Other && |
0 |
| 9167 |
"Invalid chain type"); |
--- |
9167 |
"Invalid chain type"); |
--- |
| 9168 |
bool Indexed = AM != ISD::UNINDEXED; |
0 |
9168 |
bool Indexed = AM != ISD::UNINDEXED; |
0 |
| 9169 |
assert((Indexed || Offset.isUndef()) && |
0 |
9169 |
assert((Indexed || Offset.isUndef()) && |
0 |
| 9170 |
"Unindexed masked store with an offset!"); |
--- |
9170 |
"Unindexed masked store with an offset!"); |
--- |
| 9171 |
SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other) |
0 |
9171 |
SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other) |
0 |
| 9172 |
: getVTList(MVT::Other); |
0 |
9172 |
: getVTList(MVT::Other); |
0 |
| 9173 |
SDValue Ops[] = {Chain, Val, Base, Offset, Mask}; |
0 |
9173 |
SDValue Ops[] = {Chain, Val, Base, Offset, Mask}; |
0 |
| 9174 |
FoldingSetNodeID ID; |
0 |
9174 |
FoldingSetNodeID ID; |
0 |
| 9175 |
AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops); |
0 |
9175 |
AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops); |
0 |
| 9176 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
9176 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
| 9177 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
9177 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
| 9178 |
dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO)); |
--- |
9178 |
dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO)); |
--- |
| 9179 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
9179 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
| 9180 |
ID.AddInteger(MMO->getFlags()); |
0 |
9180 |
ID.AddInteger(MMO->getFlags()); |
0 |
| 9181 |
void *IP = nullptr; |
0 |
9181 |
void *IP = nullptr; |
0 |
| 9182 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
9182 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
| 9183 |
cast(E)->refineAlignment(MMO); |
0 |
9183 |
cast(E)->refineAlignment(MMO); |
0 |
| 9184 |
return SDValue(E, 0); |
0 |
9184 |
return SDValue(E, 0); |
0 |
| 9185 |
} |
--- |
9185 |
} |
--- |
| 9186 |
auto *N = |
--- |
9186 |
auto *N = |
--- |
| 9187 |
newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, |
0 |
9187 |
newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, |
0 |
| 9188 |
IsTruncating, IsCompressing, MemVT, MMO); |
--- |
9188 |
IsTruncating, IsCompressing, MemVT, MMO); |
--- |
| 9189 |
createOperands(N, Ops); |
0 |
9189 |
createOperands(N, Ops); |
0 |
| 9190 |
|
--- |
9190 |
|
--- |
| 9191 |
CSEMap.InsertNode(N, IP); |
0 |
9191 |
CSEMap.InsertNode(N, IP); |
0 |
| 9192 |
InsertNode(N); |
0 |
9192 |
InsertNode(N); |
0 |
| 9193 |
SDValue V(N, 0); |
0 |
9193 |
SDValue V(N, 0); |
0 |
| 9194 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
9194 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 9195 |
return V; |
0 |
9195 |
return V; |
0 |
| 9196 |
} |
0 |
9196 |
} |
0 |
| 9197 |
|
--- |
9197 |
|
--- |
| 9198 |
SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, |
0 |
9198 |
SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, |
0 |
| 9199 |
SDValue Base, SDValue Offset, |
--- |
9199 |
SDValue Base, SDValue Offset, |
--- |
| 9200 |
ISD::MemIndexedMode AM) { |
--- |
9200 |
ISD::MemIndexedMode AM) { |
--- |
| 9201 |
MaskedStoreSDNode *ST = cast(OrigStore); |
0 |
9201 |
MaskedStoreSDNode *ST = cast(OrigStore); |
0 |
| 9202 |
assert(ST->getOffset().isUndef() && |
0 |
9202 |
assert(ST->getOffset().isUndef() && |
0 |
| 9203 |
"Masked store is already a indexed store!"); |
--- |
9203 |
"Masked store is already a indexed store!"); |
--- |
| 9204 |
return getMaskedStore(ST->getChain(), dl, ST->getValue(), Base, Offset, |
0 |
9204 |
return getMaskedStore(ST->getChain(), dl, ST->getValue(), Base, Offset, |
0 |
| 9205 |
ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(), |
0 |
9205 |
ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(), |
0 |
| 9206 |
AM, ST->isTruncatingStore(), ST->isCompressingStore()); |
0 |
9206 |
AM, ST->isTruncatingStore(), ST->isCompressingStore()); |
0 |
| 9207 |
} |
--- |
9207 |
} |
--- |
| 9208 |
|
--- |
9208 |
|
--- |
| 9209 |
SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, |
0 |
9209 |
SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, |
0 |
| 9210 |
ArrayRef Ops, |
--- |
9210 |
ArrayRef Ops, |
--- |
| 9211 |
MachineMemOperand *MMO, |
--- |
9211 |
MachineMemOperand *MMO, |
--- |
| 9212 |
ISD::MemIndexType IndexType, |
--- |
9212 |
ISD::MemIndexType IndexType, |
--- |
| 9213 |
ISD::LoadExtType ExtTy) { |
--- |
9213 |
ISD::LoadExtType ExtTy) { |
--- |
| 9214 |
assert(Ops.size() == 6 && "Incompatible number of operands"); |
0 |
9214 |
assert(Ops.size() == 6 && "Incompatible number of operands"); |
0 |
| 9215 |
|
--- |
9215 |
|
--- |
| 9216 |
FoldingSetNodeID ID; |
0 |
9216 |
FoldingSetNodeID ID; |
0 |
| 9217 |
AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops); |
0 |
9217 |
AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops); |
0 |
| 9218 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
9218 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
| 9219 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
9219 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
| 9220 |
dl.getIROrder(), VTs, MemVT, MMO, IndexType, ExtTy)); |
--- |
9220 |
dl.getIROrder(), VTs, MemVT, MMO, IndexType, ExtTy)); |
--- |
| 9221 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
9221 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
| 9222 |
ID.AddInteger(MMO->getFlags()); |
0 |
9222 |
ID.AddInteger(MMO->getFlags()); |
0 |
| 9223 |
void *IP = nullptr; |
0 |
9223 |
void *IP = nullptr; |
0 |
| 9224 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
9224 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
| 9225 |
cast(E)->refineAlignment(MMO); |
0 |
9225 |
cast(E)->refineAlignment(MMO); |
0 |
| 9226 |
return SDValue(E, 0); |
0 |
9226 |
return SDValue(E, 0); |
0 |
| 9227 |
} |
--- |
9227 |
} |
--- |
| 9228 |
|
--- |
9228 |
|
--- |
| 9229 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), |
0 |
9229 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), |
0 |
| 9230 |
VTs, MemVT, MMO, IndexType, ExtTy); |
--- |
9230 |
VTs, MemVT, MMO, IndexType, ExtTy); |
--- |
| 9231 |
createOperands(N, Ops); |
0 |
9231 |
createOperands(N, Ops); |
0 |
| 9232 |
|
--- |
9232 |
|
--- |
| 9233 |
assert(N->getPassThru().getValueType() == N->getValueType(0) && |
0 |
9233 |
assert(N->getPassThru().getValueType() == N->getValueType(0) && |
0 |
| 9234 |
"Incompatible type of the PassThru value in MaskedGatherSDNode"); |
--- |
9234 |
"Incompatible type of the PassThru value in MaskedGatherSDNode"); |
--- |
| 9235 |
assert(N->getMask().getValueType().getVectorElementCount() == |
0 |
9235 |
assert(N->getMask().getValueType().getVectorElementCount() == |
0 |
| 9236 |
N->getValueType(0).getVectorElementCount() && |
--- |
9236 |
N->getValueType(0).getVectorElementCount() && |
--- |
| 9237 |
"Vector width mismatch between mask and data"); |
--- |
9237 |
"Vector width mismatch between mask and data"); |
--- |
| 9238 |
assert(N->getIndex().getValueType().getVectorElementCount().isScalable() == |
0 |
9238 |
assert(N->getIndex().getValueType().getVectorElementCount().isScalable() == |
0 |
| 9239 |
N->getValueType(0).getVectorElementCount().isScalable() && |
--- |
9239 |
N->getValueType(0).getVectorElementCount().isScalable() && |
--- |
| 9240 |
"Scalable flags of index and data do not match"); |
--- |
9240 |
"Scalable flags of index and data do not match"); |
--- |
| 9241 |
assert(ElementCount::isKnownGE( |
0 |
9241 |
assert(ElementCount::isKnownGE( |
0 |
| 9242 |
N->getIndex().getValueType().getVectorElementCount(), |
--- |
9242 |
N->getIndex().getValueType().getVectorElementCount(), |
--- |
| 9243 |
N->getValueType(0).getVectorElementCount()) && |
--- |
9243 |
N->getValueType(0).getVectorElementCount()) && |
--- |
| 9244 |
"Vector width mismatch between index and data"); |
--- |
9244 |
"Vector width mismatch between index and data"); |
--- |
| 9245 |
assert(isa(N->getScale()) && |
0 |
9245 |
assert(isa(N->getScale()) && |
0 |
| 9246 |
cast(N->getScale())->getAPIntValue().isPowerOf2() && |
--- |
9246 |
cast(N->getScale())->getAPIntValue().isPowerOf2() && |
--- |
| 9247 |
"Scale should be a constant power of 2"); |
--- |
9247 |
"Scale should be a constant power of 2"); |
--- |
| 9248 |
|
--- |
9248 |
|
--- |
| 9249 |
CSEMap.InsertNode(N, IP); |
0 |
9249 |
CSEMap.InsertNode(N, IP); |
0 |
| 9250 |
InsertNode(N); |
0 |
9250 |
InsertNode(N); |
0 |
| 9251 |
SDValue V(N, 0); |
0 |
9251 |
SDValue V(N, 0); |
0 |
| 9252 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
9252 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 9253 |
return V; |
0 |
9253 |
return V; |
0 |
| 9254 |
} |
0 |
9254 |
} |
0 |
| 9255 |
|
--- |
9255 |
|
--- |
| 9256 |
SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, |
0 |
9256 |
SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, |
0 |
| 9257 |
ArrayRef Ops, |
--- |
9257 |
ArrayRef Ops, |
--- |
| 9258 |
MachineMemOperand *MMO, |
--- |
9258 |
MachineMemOperand *MMO, |
--- |
| 9259 |
ISD::MemIndexType IndexType, |
--- |
9259 |
ISD::MemIndexType IndexType, |
--- |
| 9260 |
bool IsTrunc) { |
--- |
9260 |
bool IsTrunc) { |
--- |
| 9261 |
assert(Ops.size() == 6 && "Incompatible number of operands"); |
0 |
9261 |
assert(Ops.size() == 6 && "Incompatible number of operands"); |
0 |
| 9262 |
|
--- |
9262 |
|
--- |
| 9263 |
FoldingSetNodeID ID; |
0 |
9263 |
FoldingSetNodeID ID; |
0 |
| 9264 |
AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops); |
0 |
9264 |
AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops); |
0 |
| 9265 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
9265 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
| 9266 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
9266 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
| 9267 |
dl.getIROrder(), VTs, MemVT, MMO, IndexType, IsTrunc)); |
--- |
9267 |
dl.getIROrder(), VTs, MemVT, MMO, IndexType, IsTrunc)); |
--- |
| 9268 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
9268 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
| 9269 |
ID.AddInteger(MMO->getFlags()); |
0 |
9269 |
ID.AddInteger(MMO->getFlags()); |
0 |
| 9270 |
void *IP = nullptr; |
0 |
9270 |
void *IP = nullptr; |
0 |
| 9271 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
9271 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { |
0 |
| 9272 |
cast(E)->refineAlignment(MMO); |
0 |
9272 |
cast(E)->refineAlignment(MMO); |
0 |
| 9273 |
return SDValue(E, 0); |
0 |
9273 |
return SDValue(E, 0); |
0 |
| 9274 |
} |
--- |
9274 |
} |
--- |
| 9275 |
|
--- |
9275 |
|
--- |
| 9276 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), |
0 |
9276 |
auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), |
0 |
| 9277 |
VTs, MemVT, MMO, IndexType, IsTrunc); |
--- |
9277 |
VTs, MemVT, MMO, IndexType, IsTrunc); |
--- |
| 9278 |
createOperands(N, Ops); |
0 |
9278 |
createOperands(N, Ops); |
0 |
| 9279 |
|
--- |
9279 |
|
--- |
| 9280 |
assert(N->getMask().getValueType().getVectorElementCount() == |
0 |
9280 |
assert(N->getMask().getValueType().getVectorElementCount() == |
0 |
| 9281 |
N->getValue().getValueType().getVectorElementCount() && |
--- |
9281 |
N->getValue().getValueType().getVectorElementCount() && |
--- |
| 9282 |
"Vector width mismatch between mask and data"); |
--- |
9282 |
"Vector width mismatch between mask and data"); |
--- |
| 9283 |
assert( |
0 |
9283 |
assert( |
0 |
| 9284 |
N->getIndex().getValueType().getVectorElementCount().isScalable() == |
--- |
9284 |
N->getIndex().getValueType().getVectorElementCount().isScalable() == |
--- |
| 9285 |
N->getValue().getValueType().getVectorElementCount().isScalable() && |
--- |
9285 |
N->getValue().getValueType().getVectorElementCount().isScalable() && |
--- |
| 9286 |
"Scalable flags of index and data do not match"); |
--- |
9286 |
"Scalable flags of index and data do not match"); |
--- |
| 9287 |
assert(ElementCount::isKnownGE( |
0 |
9287 |
assert(ElementCount::isKnownGE( |
0 |
| 9288 |
N->getIndex().getValueType().getVectorElementCount(), |
--- |
9288 |
N->getIndex().getValueType().getVectorElementCount(), |
--- |
| 9289 |
N->getValue().getValueType().getVectorElementCount()) && |
--- |
9289 |
N->getValue().getValueType().getVectorElementCount()) && |
--- |
| 9290 |
"Vector width mismatch between index and data"); |
--- |
9290 |
"Vector width mismatch between index and data"); |
--- |
| 9291 |
assert(isa(N->getScale()) && |
0 |
9291 |
assert(isa(N->getScale()) && |
0 |
| 9292 |
cast(N->getScale())->getAPIntValue().isPowerOf2() && |
--- |
9292 |
cast(N->getScale())->getAPIntValue().isPowerOf2() && |
--- |
| 9293 |
"Scale should be a constant power of 2"); |
--- |
9293 |
"Scale should be a constant power of 2"); |
--- |
| 9294 |
|
--- |
9294 |
|
--- |
| 9295 |
CSEMap.InsertNode(N, IP); |
0 |
9295 |
CSEMap.InsertNode(N, IP); |
0 |
| 9296 |
InsertNode(N); |
0 |
9296 |
InsertNode(N); |
0 |
| 9297 |
SDValue V(N, 0); |
0 |
9297 |
SDValue V(N, 0); |
0 |
| 9298 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
9298 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 9299 |
return V; |
0 |
9299 |
return V; |
0 |
| 9300 |
} |
0 |
9300 |
} |
0 |
| 9301 |
|
--- |
9301 |
|
--- |
| 9302 |
SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, |
0 |
9302 |
SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, |
0 |
| 9303 |
EVT MemVT, MachineMemOperand *MMO) { |
--- |
9303 |
EVT MemVT, MachineMemOperand *MMO) { |
--- |
| 9304 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
9304 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
| 9305 |
SDVTList VTs = getVTList(MVT::Other); |
0 |
9305 |
SDVTList VTs = getVTList(MVT::Other); |
0 |
| 9306 |
SDValue Ops[] = {Chain, Ptr}; |
0 |
9306 |
SDValue Ops[] = {Chain, Ptr}; |
0 |
| 9307 |
FoldingSetNodeID ID; |
0 |
9307 |
FoldingSetNodeID ID; |
0 |
| 9308 |
AddNodeIDNode(ID, ISD::GET_FPENV_MEM, VTs, Ops); |
0 |
9308 |
AddNodeIDNode(ID, ISD::GET_FPENV_MEM, VTs, Ops); |
0 |
| 9309 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
9309 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
| 9310 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
9310 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
| 9311 |
ISD::GET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO)); |
--- |
9311 |
ISD::GET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO)); |
--- |
| 9312 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
9312 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
| 9313 |
ID.AddInteger(MMO->getFlags()); |
0 |
9313 |
ID.AddInteger(MMO->getFlags()); |
0 |
| 9314 |
void *IP = nullptr; |
0 |
9314 |
void *IP = nullptr; |
0 |
| 9315 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) |
0 |
9315 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) |
0 |
| 9316 |
return SDValue(E, 0); |
0 |
9316 |
return SDValue(E, 0); |
0 |
| 9317 |
|
--- |
9317 |
|
--- |
| 9318 |
auto *N = newSDNode(ISD::GET_FPENV_MEM, dl.getIROrder(), |
0 |
9318 |
auto *N = newSDNode(ISD::GET_FPENV_MEM, dl.getIROrder(), |
0 |
| 9319 |
dl.getDebugLoc(), VTs, MemVT, MMO); |
--- |
9319 |
dl.getDebugLoc(), VTs, MemVT, MMO); |
--- |
| 9320 |
createOperands(N, Ops); |
0 |
9320 |
createOperands(N, Ops); |
0 |
| 9321 |
|
--- |
9321 |
|
--- |
| 9322 |
CSEMap.InsertNode(N, IP); |
0 |
9322 |
CSEMap.InsertNode(N, IP); |
0 |
| 9323 |
InsertNode(N); |
0 |
9323 |
InsertNode(N); |
0 |
| 9324 |
SDValue V(N, 0); |
0 |
9324 |
SDValue V(N, 0); |
0 |
| 9325 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
9325 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 9326 |
return V; |
0 |
9326 |
return V; |
0 |
| 9327 |
} |
0 |
9327 |
} |
0 |
| 9328 |
|
--- |
9328 |
|
--- |
| 9329 |
SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, |
0 |
9329 |
SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, |
0 |
| 9330 |
EVT MemVT, MachineMemOperand *MMO) { |
--- |
9330 |
EVT MemVT, MachineMemOperand *MMO) { |
--- |
| 9331 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
9331 |
assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
0 |
| 9332 |
SDVTList VTs = getVTList(MVT::Other); |
0 |
9332 |
SDVTList VTs = getVTList(MVT::Other); |
0 |
| 9333 |
SDValue Ops[] = {Chain, Ptr}; |
0 |
9333 |
SDValue Ops[] = {Chain, Ptr}; |
0 |
| 9334 |
FoldingSetNodeID ID; |
0 |
9334 |
FoldingSetNodeID ID; |
0 |
| 9335 |
AddNodeIDNode(ID, ISD::SET_FPENV_MEM, VTs, Ops); |
0 |
9335 |
AddNodeIDNode(ID, ISD::SET_FPENV_MEM, VTs, Ops); |
0 |
| 9336 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
9336 |
ID.AddInteger(MemVT.getRawBits()); |
0 |
| 9337 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
9337 |
ID.AddInteger(getSyntheticNodeSubclassData( |
0 |
| 9338 |
ISD::SET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO)); |
--- |
9338 |
ISD::SET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO)); |
--- |
| 9339 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
9339 |
ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
0 |
| 9340 |
ID.AddInteger(MMO->getFlags()); |
0 |
9340 |
ID.AddInteger(MMO->getFlags()); |
0 |
| 9341 |
void *IP = nullptr; |
0 |
9341 |
void *IP = nullptr; |
0 |
| 9342 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) |
0 |
9342 |
if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) |
0 |
| 9343 |
return SDValue(E, 0); |
0 |
9343 |
return SDValue(E, 0); |
0 |
| 9344 |
|
--- |
9344 |
|
--- |
| 9345 |
auto *N = newSDNode(ISD::SET_FPENV_MEM, dl.getIROrder(), |
0 |
9345 |
auto *N = newSDNode(ISD::SET_FPENV_MEM, dl.getIROrder(), |
0 |
| 9346 |
dl.getDebugLoc(), VTs, MemVT, MMO); |
--- |
9346 |
dl.getDebugLoc(), VTs, MemVT, MMO); |
--- |
| 9347 |
createOperands(N, Ops); |
0 |
9347 |
createOperands(N, Ops); |
0 |
| 9348 |
|
--- |
9348 |
|
--- |
| 9349 |
CSEMap.InsertNode(N, IP); |
0 |
9349 |
CSEMap.InsertNode(N, IP); |
0 |
| 9350 |
InsertNode(N); |
0 |
9350 |
InsertNode(N); |
0 |
| 9351 |
SDValue V(N, 0); |
0 |
9351 |
SDValue V(N, 0); |
0 |
| 9352 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
9352 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 9353 |
return V; |
0 |
9353 |
return V; |
0 |
| 9354 |
} |
0 |
9354 |
} |
0 |
| 9355 |
|
--- |
9355 |
|
--- |
| 9356 |
SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) { |
0 |
9356 |
SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) { |
0 |
| 9357 |
// select undef, T, F --> T (if T is a constant), otherwise F |
--- |
9357 |
// select undef, T, F --> T (if T is a constant), otherwise F |
--- |
| 9358 |
// select, ?, undef, F --> F |
--- |
9358 |
// select, ?, undef, F --> F |
--- |
| 9359 |
// select, ?, T, undef --> T |
--- |
9359 |
// select, ?, T, undef --> T |
--- |
| 9360 |
if (Cond.isUndef()) |
0 |
9360 |
if (Cond.isUndef()) |
0 |
| 9361 |
return isConstantValueOfAnyType(T) ? T : F; |
0 |
9361 |
return isConstantValueOfAnyType(T) ? T : F; |
0 |
| 9362 |
if (T.isUndef()) |
0 |
9362 |
if (T.isUndef()) |
0 |
| 9363 |
return F; |
0 |
9363 |
return F; |
0 |
| 9364 |
if (F.isUndef()) |
0 |
9364 |
if (F.isUndef()) |
0 |
| 9365 |
return T; |
0 |
9365 |
return T; |
0 |
| 9366 |
|
--- |
9366 |
|
--- |
| 9367 |
// select true, T, F --> T |
--- |
9367 |
// select true, T, F --> T |
--- |
| 9368 |
// select false, T, F --> F |
--- |
9368 |
// select false, T, F --> F |
--- |
| 9369 |
if (auto *CondC = dyn_cast(Cond)) |
0 |
9369 |
if (auto *CondC = dyn_cast(Cond)) |
0 |
| 9370 |
return CondC->isZero() ? F : T; |
0 |
9370 |
return CondC->isZero() ? F : T; |
0 |
| 9371 |
|
--- |
9371 |
|
--- |
| 9372 |
// TODO: This should simplify VSELECT with non-zero constant condition using |
--- |
9372 |
// TODO: This should simplify VSELECT with non-zero constant condition using |
--- |
| 9373 |
// something like this (but check boolean contents to be complete?): |
--- |
9373 |
// something like this (but check boolean contents to be complete?): |
--- |
| 9374 |
if (ConstantSDNode *CondC = isConstOrConstSplat(Cond, /*AllowUndefs*/ false, |
0 |
9374 |
if (ConstantSDNode *CondC = isConstOrConstSplat(Cond, /*AllowUndefs*/ false, |
0 |
| 9375 |
/*AllowTruncation*/ true)) |
--- |
9375 |
/*AllowTruncation*/ true)) |
--- |
| 9376 |
if (CondC->isZero()) |
0 |
9376 |
if (CondC->isZero()) |
0 |
| 9377 |
return F; |
0 |
9377 |
return F; |
0 |
| 9378 |
|
--- |
9378 |
|
--- |
| 9379 |
// select ?, T, T --> T |
--- |
9379 |
// select ?, T, T --> T |
--- |
| 9380 |
if (T == F) |
0 |
9380 |
if (T == F) |
0 |
| 9381 |
return T; |
0 |
9381 |
return T; |
0 |
| 9382 |
|
--- |
9382 |
|
--- |
| 9383 |
return SDValue(); |
0 |
9383 |
return SDValue(); |
0 |
| 9384 |
} |
--- |
9384 |
} |
--- |
| 9385 |
|
--- |
9385 |
|
--- |
| 9386 |
SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) { |
0 |
9386 |
SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) { |
0 |
| 9387 |
// shift undef, Y --> 0 (can always assume that the undef value is 0) |
--- |
9387 |
// shift undef, Y --> 0 (can always assume that the undef value is 0) |
--- |
| 9388 |
if (X.isUndef()) |
0 |
9388 |
if (X.isUndef()) |
0 |
| 9389 |
return getConstant(0, SDLoc(X.getNode()), X.getValueType()); |
0 |
9389 |
return getConstant(0, SDLoc(X.getNode()), X.getValueType()); |
0 |
| 9390 |
// shift X, undef --> undef (because it may shift by the bitwidth) |
--- |
9390 |
// shift X, undef --> undef (because it may shift by the bitwidth) |
--- |
| 9391 |
if (Y.isUndef()) |
0 |
9391 |
if (Y.isUndef()) |
0 |
| 9392 |
return getUNDEF(X.getValueType()); |
0 |
9392 |
return getUNDEF(X.getValueType()); |
0 |
| 9393 |
|
--- |
9393 |
|
--- |
| 9394 |
// shift 0, Y --> 0 |
--- |
9394 |
// shift 0, Y --> 0 |
--- |
| 9395 |
// shift X, 0 --> X |
--- |
9395 |
// shift X, 0 --> X |
--- |
| 9396 |
if (isNullOrNullSplat(X) || isNullOrNullSplat(Y)) |
0 |
9396 |
if (isNullOrNullSplat(X) || isNullOrNullSplat(Y)) |
0 |
| 9397 |
return X; |
0 |
9397 |
return X; |
0 |
| 9398 |
|
--- |
9398 |
|
--- |
| 9399 |
// shift X, C >= bitwidth(X) --> undef |
--- |
9399 |
// shift X, C >= bitwidth(X) --> undef |
--- |
| 9400 |
// All vector elements must be too big (or undef) to avoid partial undefs. |
--- |
9400 |
// All vector elements must be too big (or undef) to avoid partial undefs. |
--- |
| 9401 |
auto isShiftTooBig = [X](ConstantSDNode *Val) { |
0 |
9401 |
auto isShiftTooBig = [X](ConstantSDNode *Val) { |
0 |
| 9402 |
return !Val || Val->getAPIntValue().uge(X.getScalarValueSizeInBits()); |
0 |
9402 |
return !Val || Val->getAPIntValue().uge(X.getScalarValueSizeInBits()); |
0 |
| 9403 |
}; |
0 |
9403 |
}; |
0 |
| 9404 |
if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true)) |
0 |
9404 |
if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true)) |
0 |
| 9405 |
return getUNDEF(X.getValueType()); |
0 |
9405 |
return getUNDEF(X.getValueType()); |
0 |
| 9406 |
|
--- |
9406 |
|
--- |
| 9407 |
return SDValue(); |
0 |
9407 |
return SDValue(); |
0 |
| 9408 |
} |
--- |
9408 |
} |
--- |
| 9409 |
|
--- |
9409 |
|
--- |
| 9410 |
SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, |
0 |
9410 |
SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, |
0 |
| 9411 |
SDNodeFlags Flags) { |
--- |
9411 |
SDNodeFlags Flags) { |
--- |
| 9412 |
// If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand |
--- |
9412 |
// If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand |
--- |
| 9413 |
// (an undef operand can be chosen to be Nan/Inf), then the result of this |
--- |
9413 |
// (an undef operand can be chosen to be Nan/Inf), then the result of this |
--- |
| 9414 |
// operation is poison. That result can be relaxed to undef. |
--- |
9414 |
// operation is poison. That result can be relaxed to undef. |
--- |
| 9415 |
ConstantFPSDNode *XC = isConstOrConstSplatFP(X, /* AllowUndefs */ true); |
0 |
9415 |
ConstantFPSDNode *XC = isConstOrConstSplatFP(X, /* AllowUndefs */ true); |
0 |
| 9416 |
ConstantFPSDNode *YC = isConstOrConstSplatFP(Y, /* AllowUndefs */ true); |
0 |
9416 |
ConstantFPSDNode *YC = isConstOrConstSplatFP(Y, /* AllowUndefs */ true); |
0 |
| 9417 |
bool HasNan = (XC && XC->getValueAPF().isNaN()) || |
0 |
9417 |
bool HasNan = (XC && XC->getValueAPF().isNaN()) || |
0 |
| 9418 |
(YC && YC->getValueAPF().isNaN()); |
0 |
9418 |
(YC && YC->getValueAPF().isNaN()); |
0 |
| 9419 |
bool HasInf = (XC && XC->getValueAPF().isInfinity()) || |
0 |
9419 |
bool HasInf = (XC && XC->getValueAPF().isInfinity()) || |
0 |
| 9420 |
(YC && YC->getValueAPF().isInfinity()); |
0 |
9420 |
(YC && YC->getValueAPF().isInfinity()); |
0 |
| 9421 |
|
--- |
9421 |
|
--- |
| 9422 |
if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef())) |
0 |
9422 |
if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef())) |
0 |
| 9423 |
return getUNDEF(X.getValueType()); |
0 |
9423 |
return getUNDEF(X.getValueType()); |
0 |
| 9424 |
|
--- |
9424 |
|
--- |
| 9425 |
if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef())) |
0 |
9425 |
if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef())) |
0 |
| 9426 |
return getUNDEF(X.getValueType()); |
0 |
9426 |
return getUNDEF(X.getValueType()); |
0 |
| 9427 |
|
--- |
9427 |
|
--- |
| 9428 |
if (!YC) |
0 |
9428 |
if (!YC) |
0 |
| 9429 |
return SDValue(); |
0 |
9429 |
return SDValue(); |
0 |
| 9430 |
|
--- |
9430 |
|
--- |
| 9431 |
// X + -0.0 --> X |
--- |
9431 |
// X + -0.0 --> X |
--- |
| 9432 |
if (Opcode == ISD::FADD) |
0 |
9432 |
if (Opcode == ISD::FADD) |
0 |
| 9433 |
if (YC->getValueAPF().isNegZero()) |
0 |
9433 |
if (YC->getValueAPF().isNegZero()) |
0 |
| 9434 |
return X; |
0 |
9434 |
return X; |
0 |
| 9435 |
|
--- |
9435 |
|
--- |
| 9436 |
// X - +0.0 --> X |
--- |
9436 |
// X - +0.0 --> X |
--- |
| 9437 |
if (Opcode == ISD::FSUB) |
0 |
9437 |
if (Opcode == ISD::FSUB) |
0 |
| 9438 |
if (YC->getValueAPF().isPosZero()) |
0 |
9438 |
if (YC->getValueAPF().isPosZero()) |
0 |
| 9439 |
return X; |
0 |
9439 |
return X; |
0 |
| 9440 |
|
--- |
9440 |
|
--- |
| 9441 |
// X * 1.0 --> X |
--- |
9441 |
// X * 1.0 --> X |
--- |
| 9442 |
// X / 1.0 --> X |
--- |
9442 |
// X / 1.0 --> X |
--- |
| 9443 |
if (Opcode == ISD::FMUL || Opcode == ISD::FDIV) |
0 |
9443 |
if (Opcode == ISD::FMUL || Opcode == ISD::FDIV) |
0 |
| 9444 |
if (YC->getValueAPF().isExactlyValue(1.0)) |
0 |
9444 |
if (YC->getValueAPF().isExactlyValue(1.0)) |
0 |
| 9445 |
return X; |
0 |
9445 |
return X; |
0 |
| 9446 |
|
--- |
9446 |
|
--- |
| 9447 |
// X * 0.0 --> 0.0 |
--- |
9447 |
// X * 0.0 --> 0.0 |
--- |
| 9448 |
if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros()) |
0 |
9448 |
if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros()) |
0 |
| 9449 |
if (YC->getValueAPF().isZero()) |
0 |
9449 |
if (YC->getValueAPF().isZero()) |
0 |
| 9450 |
return getConstantFP(0.0, SDLoc(Y), Y.getValueType()); |
0 |
9450 |
return getConstantFP(0.0, SDLoc(Y), Y.getValueType()); |
0 |
| 9451 |
|
--- |
9451 |
|
--- |
| 9452 |
return SDValue(); |
0 |
9452 |
return SDValue(); |
0 |
| 9453 |
} |
--- |
9453 |
} |
--- |
| 9454 |
|
--- |
9454 |
|
--- |
| 9455 |
SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, |
0 |
9455 |
SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, |
0 |
| 9456 |
SDValue Ptr, SDValue SV, unsigned Align) { |
--- |
9456 |
SDValue Ptr, SDValue SV, unsigned Align) { |
--- |
| 9457 |
SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) }; |
0 |
9457 |
SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) }; |
0 |
| 9458 |
return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops); |
0 |
9458 |
return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops); |
0 |
| 9459 |
} |
--- |
9459 |
} |
--- |
| 9460 |
|
--- |
9460 |
|
--- |
| 9461 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
0 |
9461 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
0 |
| 9462 |
ArrayRef Ops) { |
--- |
9462 |
ArrayRef Ops) { |
--- |
| 9463 |
switch (Ops.size()) { |
0 |
9463 |
switch (Ops.size()) { |
0 |
| 9464 |
case 0: return getNode(Opcode, DL, VT); |
0 |
9464 |
case 0: return getNode(Opcode, DL, VT); |
0 |
| 9465 |
case 1: return getNode(Opcode, DL, VT, static_cast(Ops[0])); |
0 |
9465 |
case 1: return getNode(Opcode, DL, VT, static_cast(Ops[0])); |
0 |
| 9466 |
case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); |
0 |
9466 |
case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); |
0 |
| 9467 |
case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); |
0 |
9467 |
case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); |
0 |
| 9468 |
default: break; |
0 |
9468 |
default: break; |
0 |
| 9469 |
} |
--- |
9469 |
} |
--- |
| 9470 |
|
--- |
9470 |
|
--- |
| 9471 |
// Copy from an SDUse array into an SDValue array for use with |
--- |
9471 |
// Copy from an SDUse array into an SDValue array for use with |
--- |
| 9472 |
// the regular getNode logic. |
--- |
9472 |
// the regular getNode logic. |
--- |
| 9473 |
SmallVector NewOps(Ops.begin(), Ops.end()); |
0 |
9473 |
SmallVector NewOps(Ops.begin(), Ops.end()); |
0 |
| 9474 |
return getNode(Opcode, DL, VT, NewOps); |
0 |
9474 |
return getNode(Opcode, DL, VT, NewOps); |
0 |
| 9475 |
} |
--- |
9475 |
} |
--- |
| 9476 |
|
--- |
9476 |
|
--- |
| 9477 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
10 |
9477 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
10 |
| 9478 |
ArrayRef Ops) { |
--- |
9478 |
ArrayRef Ops) { |
--- |
| 9479 |
SDNodeFlags Flags; |
10 |
9479 |
SDNodeFlags Flags; |
10 |
| 9480 |
if (Inserter) |
10 |
9480 |
if (Inserter) |
10 |
| 9481 |
Flags = Inserter->getFlags(); |
0 |
9481 |
Flags = Inserter->getFlags(); |
0 |
| 9482 |
return getNode(Opcode, DL, VT, Ops, Flags); |
10 |
9482 |
return getNode(Opcode, DL, VT, Ops, Flags); |
10 |
| 9483 |
} |
--- |
9483 |
} |
--- |
| 9484 |
|
--- |
9484 |
|
--- |
| 9485 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
15 |
9485 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
15 |
| 9486 |
ArrayRef Ops, const SDNodeFlags Flags) { |
--- |
9486 |
ArrayRef Ops, const SDNodeFlags Flags) { |
--- |
| 9487 |
unsigned NumOps = Ops.size(); |
15 |
9487 |
unsigned NumOps = Ops.size(); |
15 |
| 9488 |
switch (NumOps) { |
15 |
9488 |
switch (NumOps) { |
15 |
| 9489 |
case 0: return getNode(Opcode, DL, VT); |
0 |
9489 |
case 0: return getNode(Opcode, DL, VT); |
0 |
| 9490 |
case 1: return getNode(Opcode, DL, VT, Ops[0], Flags); |
14 |
9490 |
case 1: return getNode(Opcode, DL, VT, Ops[0], Flags); |
14 |
| 9491 |
case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags); |
0 |
9491 |
case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags); |
0 |
| 9492 |
case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2], Flags); |
1 |
9492 |
case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2], Flags); |
1 |
| 9493 |
default: break; |
0 |
9493 |
default: break; |
0 |
| 9494 |
} |
--- |
9494 |
} |
--- |
| 9495 |
|
--- |
9495 |
|
--- |
| 9496 |
#ifndef NDEBUG |
--- |
9496 |
#ifndef NDEBUG |
--- |
| 9497 |
for (const auto &Op : Ops) |
0 |
9497 |
for (const auto &Op : Ops) |
0 |
| 9498 |
assert(Op.getOpcode() != ISD::DELETED_NODE && |
0 |
9498 |
assert(Op.getOpcode() != ISD::DELETED_NODE && |
0 |
| 9499 |
"Operand is DELETED_NODE!"); |
--- |
9499 |
"Operand is DELETED_NODE!"); |
--- |
| 9500 |
#endif |
--- |
9500 |
#endif |
--- |
| 9501 |
|
--- |
9501 |
|
--- |
| 9502 |
switch (Opcode) { |
0 |
9502 |
switch (Opcode) { |
0 |
| 9503 |
default: break; |
0 |
9503 |
default: break; |
0 |
| 9504 |
case ISD::BUILD_VECTOR: |
0 |
9504 |
case ISD::BUILD_VECTOR: |
0 |
| 9505 |
// Attempt to simplify BUILD_VECTOR. |
--- |
9505 |
// Attempt to simplify BUILD_VECTOR. |
--- |
| 9506 |
if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this)) |
0 |
9506 |
if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this)) |
0 |
| 9507 |
return V; |
0 |
9507 |
return V; |
0 |
| 9508 |
break; |
0 |
9508 |
break; |
0 |
| 9509 |
case ISD::CONCAT_VECTORS: |
0 |
9509 |
case ISD::CONCAT_VECTORS: |
0 |
| 9510 |
if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this)) |
0 |
9510 |
if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this)) |
0 |
| 9511 |
return V; |
0 |
9511 |
return V; |
0 |
| 9512 |
break; |
0 |
9512 |
break; |
0 |
| 9513 |
case ISD::SELECT_CC: |
0 |
9513 |
case ISD::SELECT_CC: |
0 |
| 9514 |
assert(NumOps == 5 && "SELECT_CC takes 5 operands!"); |
0 |
9514 |
assert(NumOps == 5 && "SELECT_CC takes 5 operands!"); |
0 |
| 9515 |
assert(Ops[0].getValueType() == Ops[1].getValueType() && |
0 |
9515 |
assert(Ops[0].getValueType() == Ops[1].getValueType() && |
0 |
| 9516 |
"LHS and RHS of condition must have same type!"); |
--- |
9516 |
"LHS and RHS of condition must have same type!"); |
--- |
| 9517 |
assert(Ops[2].getValueType() == Ops[3].getValueType() && |
0 |
9517 |
assert(Ops[2].getValueType() == Ops[3].getValueType() && |
0 |
| 9518 |
"True and False arms of SelectCC must have same type!"); |
--- |
9518 |
"True and False arms of SelectCC must have same type!"); |
--- |
| 9519 |
assert(Ops[2].getValueType() == VT && |
0 |
9519 |
assert(Ops[2].getValueType() == VT && |
0 |
| 9520 |
"select_cc node must be of same type as true and false value!"); |
--- |
9520 |
"select_cc node must be of same type as true and false value!"); |
--- |
| 9521 |
assert((!Ops[0].getValueType().isVector() || |
0 |
9521 |
assert((!Ops[0].getValueType().isVector() || |
0 |
| 9522 |
Ops[0].getValueType().getVectorElementCount() == |
--- |
9522 |
Ops[0].getValueType().getVectorElementCount() == |
--- |
| 9523 |
VT.getVectorElementCount()) && |
--- |
9523 |
VT.getVectorElementCount()) && |
--- |
| 9524 |
"Expected select_cc with vector result to have the same sized " |
--- |
9524 |
"Expected select_cc with vector result to have the same sized " |
--- |
| 9525 |
"comparison type!"); |
--- |
9525 |
"comparison type!"); |
--- |
| 9526 |
break; |
0 |
9526 |
break; |
0 |
| 9527 |
case ISD::BR_CC: |
0 |
9527 |
case ISD::BR_CC: |
0 |
| 9528 |
assert(NumOps == 5 && "BR_CC takes 5 operands!"); |
0 |
9528 |
assert(NumOps == 5 && "BR_CC takes 5 operands!"); |
0 |
| 9529 |
assert(Ops[2].getValueType() == Ops[3].getValueType() && |
0 |
9529 |
assert(Ops[2].getValueType() == Ops[3].getValueType() && |
0 |
| 9530 |
"LHS/RHS of comparison should match types!"); |
--- |
9530 |
"LHS/RHS of comparison should match types!"); |
--- |
| 9531 |
break; |
0 |
9531 |
break; |
0 |
| 9532 |
case ISD::VP_ADD: |
0 |
9532 |
case ISD::VP_ADD: |
0 |
| 9533 |
case ISD::VP_SUB: |
--- |
9533 |
case ISD::VP_SUB: |
--- |
| 9534 |
// If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR |
--- |
9534 |
// If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR |
--- |
| 9535 |
if (VT.isVector() && VT.getVectorElementType() == MVT::i1) |
0 |
9535 |
if (VT.isVector() && VT.getVectorElementType() == MVT::i1) |
0 |
| 9536 |
Opcode = ISD::VP_XOR; |
0 |
9536 |
Opcode = ISD::VP_XOR; |
0 |
| 9537 |
break; |
0 |
9537 |
break; |
0 |
| 9538 |
case ISD::VP_MUL: |
0 |
9538 |
case ISD::VP_MUL: |
0 |
| 9539 |
// If it is VP_MUL mask operation then turn it to VP_AND |
--- |
9539 |
// If it is VP_MUL mask operation then turn it to VP_AND |
--- |
| 9540 |
if (VT.isVector() && VT.getVectorElementType() == MVT::i1) |
0 |
9540 |
if (VT.isVector() && VT.getVectorElementType() == MVT::i1) |
0 |
| 9541 |
Opcode = ISD::VP_AND; |
0 |
9541 |
Opcode = ISD::VP_AND; |
0 |
| 9542 |
break; |
0 |
9542 |
break; |
0 |
| 9543 |
case ISD::VP_REDUCE_MUL: |
0 |
9543 |
case ISD::VP_REDUCE_MUL: |
0 |
| 9544 |
// If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND |
--- |
9544 |
// If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND |
--- |
| 9545 |
if (VT == MVT::i1) |
0 |
9545 |
if (VT == MVT::i1) |
0 |
| 9546 |
Opcode = ISD::VP_REDUCE_AND; |
0 |
9546 |
Opcode = ISD::VP_REDUCE_AND; |
0 |
| 9547 |
break; |
0 |
9547 |
break; |
0 |
| 9548 |
case ISD::VP_REDUCE_ADD: |
0 |
9548 |
case ISD::VP_REDUCE_ADD: |
0 |
| 9549 |
// If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR |
--- |
9549 |
// If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR |
--- |
| 9550 |
if (VT == MVT::i1) |
0 |
9550 |
if (VT == MVT::i1) |
0 |
| 9551 |
Opcode = ISD::VP_REDUCE_XOR; |
0 |
9551 |
Opcode = ISD::VP_REDUCE_XOR; |
0 |
| 9552 |
break; |
0 |
9552 |
break; |
0 |
| 9553 |
case ISD::VP_REDUCE_SMAX: |
0 |
9553 |
case ISD::VP_REDUCE_SMAX: |
0 |
| 9554 |
case ISD::VP_REDUCE_UMIN: |
--- |
9554 |
case ISD::VP_REDUCE_UMIN: |
--- |
| 9555 |
// If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to |
--- |
9555 |
// If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to |
--- |
| 9556 |
// VP_REDUCE_AND. |
--- |
9556 |
// VP_REDUCE_AND. |
--- |
| 9557 |
if (VT == MVT::i1) |
0 |
9557 |
if (VT == MVT::i1) |
0 |
| 9558 |
Opcode = ISD::VP_REDUCE_AND; |
0 |
9558 |
Opcode = ISD::VP_REDUCE_AND; |
0 |
| 9559 |
break; |
0 |
9559 |
break; |
0 |
| 9560 |
case ISD::VP_REDUCE_SMIN: |
0 |
9560 |
case ISD::VP_REDUCE_SMIN: |
0 |
| 9561 |
case ISD::VP_REDUCE_UMAX: |
--- |
9561 |
case ISD::VP_REDUCE_UMAX: |
--- |
| 9562 |
// If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to |
--- |
9562 |
// If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to |
--- |
| 9563 |
// VP_REDUCE_OR. |
--- |
9563 |
// VP_REDUCE_OR. |
--- |
| 9564 |
if (VT == MVT::i1) |
0 |
9564 |
if (VT == MVT::i1) |
0 |
| 9565 |
Opcode = ISD::VP_REDUCE_OR; |
0 |
9565 |
Opcode = ISD::VP_REDUCE_OR; |
0 |
| 9566 |
break; |
0 |
9566 |
break; |
0 |
| 9567 |
} |
--- |
9567 |
} |
--- |
| 9568 |
|
--- |
9568 |
|
--- |
| 9569 |
// Memoize nodes. |
--- |
9569 |
// Memoize nodes. |
--- |
| 9570 |
SDNode *N; |
--- |
9570 |
SDNode *N; |
--- |
| 9571 |
SDVTList VTs = getVTList(VT); |
0 |
9571 |
SDVTList VTs = getVTList(VT); |
0 |
| 9572 |
|
--- |
9572 |
|
--- |
| 9573 |
if (VT != MVT::Glue) { |
0 |
9573 |
if (VT != MVT::Glue) { |
0 |
| 9574 |
FoldingSetNodeID ID; |
0 |
9574 |
FoldingSetNodeID ID; |
0 |
| 9575 |
AddNodeIDNode(ID, Opcode, VTs, Ops); |
0 |
9575 |
AddNodeIDNode(ID, Opcode, VTs, Ops); |
0 |
| 9576 |
void *IP = nullptr; |
0 |
9576 |
void *IP = nullptr; |
0 |
| 9577 |
|
--- |
9577 |
|
--- |
| 9578 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) |
0 |
9578 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) |
0 |
| 9579 |
return SDValue(E, 0); |
0 |
9579 |
return SDValue(E, 0); |
0 |
| 9580 |
|
--- |
9580 |
|
--- |
| 9581 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
0 |
9581 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
0 |
| 9582 |
createOperands(N, Ops); |
0 |
9582 |
createOperands(N, Ops); |
0 |
| 9583 |
|
--- |
9583 |
|
--- |
| 9584 |
CSEMap.InsertNode(N, IP); |
0 |
9584 |
CSEMap.InsertNode(N, IP); |
0 |
| 9585 |
} else { |
0 |
9585 |
} else { |
0 |
| 9586 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
0 |
9586 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
0 |
| 9587 |
createOperands(N, Ops); |
0 |
9587 |
createOperands(N, Ops); |
0 |
| 9588 |
} |
--- |
9588 |
} |
--- |
| 9589 |
|
--- |
9589 |
|
--- |
| 9590 |
N->setFlags(Flags); |
0 |
9590 |
N->setFlags(Flags); |
0 |
| 9591 |
InsertNode(N); |
0 |
9591 |
InsertNode(N); |
0 |
| 9592 |
SDValue V(N, 0); |
0 |
9592 |
SDValue V(N, 0); |
0 |
| 9593 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
9593 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
0 |
| 9594 |
return V; |
0 |
9594 |
return V; |
0 |
| 9595 |
} |
--- |
9595 |
} |
--- |
| 9596 |
|
--- |
9596 |
|
--- |
| 9597 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, |
0 |
9597 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, |
0 |
| 9598 |
ArrayRef ResultTys, ArrayRef Ops) { |
--- |
9598 |
ArrayRef ResultTys, ArrayRef Ops) { |
--- |
| 9599 |
return getNode(Opcode, DL, getVTList(ResultTys), Ops); |
0 |
9599 |
return getNode(Opcode, DL, getVTList(ResultTys), Ops); |
0 |
| 9600 |
} |
--- |
9600 |
} |
--- |
| 9601 |
|
--- |
9601 |
|
--- |
| 9602 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
8 |
9602 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
8 |
| 9603 |
ArrayRef Ops) { |
--- |
9603 |
ArrayRef Ops) { |
--- |
| 9604 |
SDNodeFlags Flags; |
8 |
9604 |
SDNodeFlags Flags; |
8 |
| 9605 |
if (Inserter) |
8 |
9605 |
if (Inserter) |
8 |
| 9606 |
Flags = Inserter->getFlags(); |
0 |
9606 |
Flags = Inserter->getFlags(); |
0 |
| 9607 |
return getNode(Opcode, DL, VTList, Ops, Flags); |
8 |
9607 |
return getNode(Opcode, DL, VTList, Ops, Flags); |
8 |
| 9608 |
} |
--- |
9608 |
} |
--- |
| 9609 |
|
--- |
9609 |
|
--- |
| 9610 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
8 |
9610 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
8 |
| 9611 |
ArrayRef Ops, const SDNodeFlags Flags) { |
--- |
9611 |
ArrayRef Ops, const SDNodeFlags Flags) { |
--- |
| 9612 |
if (VTList.NumVTs == 1) |
8 |
9612 |
if (VTList.NumVTs == 1) |
8 |
| 9613 |
return getNode(Opcode, DL, VTList.VTs[0], Ops, Flags); |
5 |
9613 |
return getNode(Opcode, DL, VTList.VTs[0], Ops, Flags); |
5 |
| 9614 |
|
--- |
9614 |
|
--- |
| 9615 |
#ifndef NDEBUG |
--- |
9615 |
#ifndef NDEBUG |
--- |
| 9616 |
for (const auto &Op : Ops) |
10 |
9616 |
for (const auto &Op : Ops) |
10 |
| 9617 |
assert(Op.getOpcode() != ISD::DELETED_NODE && |
7 |
9617 |
assert(Op.getOpcode() != ISD::DELETED_NODE && |
7 |
| 9618 |
"Operand is DELETED_NODE!"); |
--- |
9618 |
"Operand is DELETED_NODE!"); |
--- |
| 9619 |
#endif |
--- |
9619 |
#endif |
--- |
| 9620 |
|
--- |
9620 |
|
--- |
| 9621 |
switch (Opcode) { |
3 |
9621 |
switch (Opcode) { |
3 |
| 9622 |
case ISD::SADDO: |
0 |
9622 |
case ISD::SADDO: |
0 |
| 9623 |
case ISD::UADDO: |
--- |
9623 |
case ISD::UADDO: |
--- |
| 9624 |
case ISD::SSUBO: |
--- |
9624 |
case ISD::SSUBO: |
--- |
| 9625 |
case ISD::USUBO: { |
--- |
9625 |
case ISD::USUBO: { |
--- |
| 9626 |
assert(VTList.NumVTs == 2 && Ops.size() == 2 && |
0 |
9626 |
assert(VTList.NumVTs == 2 && Ops.size() == 2 && |
0 |
| 9627 |
"Invalid add/sub overflow op!"); |
--- |
9627 |
"Invalid add/sub overflow op!"); |
--- |
| 9628 |
assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() && |
0 |
9628 |
assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() && |
0 |
| 9629 |
Ops[0].getValueType() == Ops[1].getValueType() && |
--- |
9629 |
Ops[0].getValueType() == Ops[1].getValueType() && |
--- |
| 9630 |
Ops[0].getValueType() == VTList.VTs[0] && |
--- |
9630 |
Ops[0].getValueType() == VTList.VTs[0] && |
--- |
| 9631 |
"Binary operator types must match!"); |
--- |
9631 |
"Binary operator types must match!"); |
--- |
| 9632 |
SDValue N1 = Ops[0], N2 = Ops[1]; |
0 |
9632 |
SDValue N1 = Ops[0], N2 = Ops[1]; |
0 |
| 9633 |
canonicalizeCommutativeBinop(Opcode, N1, N2); |
0 |
9633 |
canonicalizeCommutativeBinop(Opcode, N1, N2); |
0 |
| 9634 |
|
--- |
9634 |
|
--- |
| 9635 |
// (X +- 0) -> X with zero-overflow. |
--- |
9635 |
// (X +- 0) -> X with zero-overflow. |
--- |
| 9636 |
ConstantSDNode *N2CV = isConstOrConstSplat(N2, /*AllowUndefs*/ false, |
0 |
9636 |
ConstantSDNode *N2CV = isConstOrConstSplat(N2, /*AllowUndefs*/ false, |
0 |
| 9637 |
/*AllowTruncation*/ true); |
--- |
9637 |
/*AllowTruncation*/ true); |
--- |
| 9638 |
if (N2CV && N2CV->isZero()) { |
0 |
9638 |
if (N2CV && N2CV->isZero()) { |
0 |
| 9639 |
SDValue ZeroOverFlow = getConstant(0, DL, VTList.VTs[1]); |
0 |
9639 |
SDValue ZeroOverFlow = getConstant(0, DL, VTList.VTs[1]); |
0 |
| 9640 |
return getNode(ISD::MERGE_VALUES, DL, VTList, {N1, ZeroOverFlow}, Flags); |
0 |
9640 |
return getNode(ISD::MERGE_VALUES, DL, VTList, {N1, ZeroOverFlow}, Flags); |
0 |
| 9641 |
} |
--- |
9641 |
} |
--- |
| 9642 |
break; |
0 |
9642 |
break; |
0 |
| 9643 |
} |
--- |
9643 |
} |
--- |
| 9644 |
case ISD::SMUL_LOHI: |
0 |
9644 |
case ISD::SMUL_LOHI: |
0 |
| 9645 |
case ISD::UMUL_LOHI: { |
--- |
9645 |
case ISD::UMUL_LOHI: { |
--- |
| 9646 |
assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!"); |
0 |
9646 |
assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!"); |
0 |
| 9647 |
assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] && |
0 |
9647 |
assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] && |
0 |
| 9648 |
VTList.VTs[0] == Ops[0].getValueType() && |
--- |
9648 |
VTList.VTs[0] == Ops[0].getValueType() && |
--- |
| 9649 |
VTList.VTs[0] == Ops[1].getValueType() && |
--- |
9649 |
VTList.VTs[0] == Ops[1].getValueType() && |
--- |
| 9650 |
"Binary operator types must match!"); |
--- |
9650 |
"Binary operator types must match!"); |
--- |
| 9651 |
break; |
0 |
9651 |
break; |
0 |
| 9652 |
} |
--- |
9652 |
} |
--- |
| 9653 |
case ISD::FFREXP: { |
0 |
9653 |
case ISD::FFREXP: { |
0 |
| 9654 |
assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!"); |
0 |
9654 |
assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!"); |
0 |
| 9655 |
assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() && |
0 |
9655 |
assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() && |
0 |
| 9656 |
VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch"); |
--- |
9656 |
VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch"); |
--- |
| 9657 |
|
--- |
9657 |
|
--- |
| 9658 |
if (const ConstantFPSDNode *C = dyn_cast(Ops[0])) { |
0 |
9658 |
if (const ConstantFPSDNode *C = dyn_cast(Ops[0])) { |
0 |
| 9659 |
int FrexpExp; |
--- |
9659 |
int FrexpExp; |
--- |
| 9660 |
APFloat FrexpMant = |
--- |
9660 |
APFloat FrexpMant = |
--- |
| 9661 |
frexp(C->getValueAPF(), FrexpExp, APFloat::rmNearestTiesToEven); |
0 |
9661 |
frexp(C->getValueAPF(), FrexpExp, APFloat::rmNearestTiesToEven); |
0 |
| 9662 |
SDValue Result0 = getConstantFP(FrexpMant, DL, VTList.VTs[0]); |
0 |
9662 |
SDValue Result0 = getConstantFP(FrexpMant, DL, VTList.VTs[0]); |
0 |
| 9663 |
SDValue Result1 = |
--- |
9663 |
SDValue Result1 = |
--- |
| 9664 |
getConstant(FrexpMant.isFinite() ? FrexpExp : 0, DL, VTList.VTs[1]); |
0 |
9664 |
getConstant(FrexpMant.isFinite() ? FrexpExp : 0, DL, VTList.VTs[1]); |
0 |
| 9665 |
return getNode(ISD::MERGE_VALUES, DL, VTList, {Result0, Result1}, Flags); |
0 |
9665 |
return getNode(ISD::MERGE_VALUES, DL, VTList, {Result0, Result1}, Flags); |
0 |
| 9666 |
} |
0 |
9666 |
} |
0 |
| 9667 |
|
--- |
9667 |
|
--- |
| 9668 |
break; |
0 |
9668 |
break; |
0 |
| 9669 |
} |
--- |
9669 |
} |
--- |
| 9670 |
case ISD::STRICT_FP_EXTEND: |
0 |
9670 |
case ISD::STRICT_FP_EXTEND: |
0 |
| 9671 |
assert(VTList.NumVTs == 2 && Ops.size() == 2 && |
0 |
9671 |
assert(VTList.NumVTs == 2 && Ops.size() == 2 && |
0 |
| 9672 |
"Invalid STRICT_FP_EXTEND!"); |
--- |
9672 |
"Invalid STRICT_FP_EXTEND!"); |
--- |
| 9673 |
assert(VTList.VTs[0].isFloatingPoint() && |
0 |
9673 |
assert(VTList.VTs[0].isFloatingPoint() && |
0 |
| 9674 |
Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!"); |
--- |
9674 |
Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!"); |
--- |
| 9675 |
assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && |
0 |
9675 |
assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && |
0 |
| 9676 |
"STRICT_FP_EXTEND result type should be vector iff the operand " |
--- |
9676 |
"STRICT_FP_EXTEND result type should be vector iff the operand " |
--- |
| 9677 |
"type is vector!"); |
--- |
9677 |
"type is vector!"); |
--- |
| 9678 |
assert((!VTList.VTs[0].isVector() || |
0 |
9678 |
assert((!VTList.VTs[0].isVector() || |
0 |
| 9679 |
VTList.VTs[0].getVectorElementCount() == |
--- |
9679 |
VTList.VTs[0].getVectorElementCount() == |
--- |
| 9680 |
Ops[1].getValueType().getVectorElementCount()) && |
--- |
9680 |
Ops[1].getValueType().getVectorElementCount()) && |
--- |
| 9681 |
"Vector element count mismatch!"); |
--- |
9681 |
"Vector element count mismatch!"); |
--- |
| 9682 |
assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) && |
0 |
9682 |
assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) && |
0 |
| 9683 |
"Invalid fpext node, dst <= src!"); |
--- |
9683 |
"Invalid fpext node, dst <= src!"); |
--- |
| 9684 |
break; |
0 |
9684 |
break; |
0 |
| 9685 |
case ISD::STRICT_FP_ROUND: |
0 |
9685 |
case ISD::STRICT_FP_ROUND: |
0 |
| 9686 |
assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!"); |
0 |
9686 |
assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!"); |
0 |
| 9687 |
assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && |
0 |
9687 |
assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() && |
0 |
| 9688 |
"STRICT_FP_ROUND result type should be vector iff the operand " |
--- |
9688 |
"STRICT_FP_ROUND result type should be vector iff the operand " |
--- |
| 9689 |
"type is vector!"); |
--- |
9689 |
"type is vector!"); |
--- |
| 9690 |
assert((!VTList.VTs[0].isVector() || |
0 |
9690 |
assert((!VTList.VTs[0].isVector() || |
0 |
| 9691 |
VTList.VTs[0].getVectorElementCount() == |
--- |
9691 |
VTList.VTs[0].getVectorElementCount() == |
--- |
| 9692 |
Ops[1].getValueType().getVectorElementCount()) && |
--- |
9692 |
Ops[1].getValueType().getVectorElementCount()) && |
--- |
| 9693 |
"Vector element count mismatch!"); |
--- |
9693 |
"Vector element count mismatch!"); |
--- |
| 9694 |
assert(VTList.VTs[0].isFloatingPoint() && |
0 |
9694 |
assert(VTList.VTs[0].isFloatingPoint() && |
0 |
| 9695 |
Ops[1].getValueType().isFloatingPoint() && |
--- |
9695 |
Ops[1].getValueType().isFloatingPoint() && |
--- |
| 9696 |
VTList.VTs[0].bitsLT(Ops[1].getValueType()) && |
--- |
9696 |
VTList.VTs[0].bitsLT(Ops[1].getValueType()) && |
--- |
| 9697 |
isa(Ops[2]) && |
--- |
9697 |
isa(Ops[2]) && |
--- |
| 9698 |
(cast(Ops[2])->getZExtValue() == 0 || |
--- |
9698 |
(cast(Ops[2])->getZExtValue() == 0 || |
--- |
| 9699 |
cast(Ops[2])->getZExtValue() == 1) && |
--- |
9699 |
cast(Ops[2])->getZExtValue() == 1) && |
--- |
| 9700 |
"Invalid STRICT_FP_ROUND!"); |
--- |
9700 |
"Invalid STRICT_FP_ROUND!"); |
--- |
| 9701 |
break; |
0 |
9701 |
break; |
0 |
| 9702 |
#if 0 |
--- |
9702 |
#if 0 |
--- |
| 9703 |
// FIXME: figure out how to safely handle things like |
--- |
9703 |
// FIXME: figure out how to safely handle things like |
--- |
| 9704 |
// int foo(int x) { return 1 << (x & 255); } |
--- |
9704 |
// int foo(int x) { return 1 << (x & 255); } |
--- |
| 9705 |
// int bar() { return foo(256); } |
--- |
9705 |
// int bar() { return foo(256); } |
--- |
| 9706 |
case ISD::SRA_PARTS: |
--- |
9706 |
case ISD::SRA_PARTS: |
--- |
| 9707 |
case ISD::SRL_PARTS: |
--- |
9707 |
case ISD::SRL_PARTS: |
--- |
| 9708 |
case ISD::SHL_PARTS: |
--- |
9708 |
case ISD::SHL_PARTS: |
--- |
| 9709 |
if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && |
--- |
9709 |
if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && |
--- |
| 9710 |
cast(N3.getOperand(1))->getVT() != MVT::i1) |
--- |
9710 |
cast(N3.getOperand(1))->getVT() != MVT::i1) |
--- |
| 9711 |
return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); |
--- |
9711 |
return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); |
--- |
| 9712 |
else if (N3.getOpcode() == ISD::AND) |
--- |
9712 |
else if (N3.getOpcode() == ISD::AND) |
--- |
| 9713 |
if (ConstantSDNode *AndRHS = dyn_cast(N3.getOperand(1))) { |
--- |
9713 |
if (ConstantSDNode *AndRHS = dyn_cast(N3.getOperand(1))) { |
--- |
| 9714 |
// If the and is only masking out bits that cannot effect the shift, |
--- |
9714 |
// If the and is only masking out bits that cannot effect the shift, |
--- |
| 9715 |
// eliminate the and. |
--- |
9715 |
// eliminate the and. |
--- |
| 9716 |
unsigned NumBits = VT.getScalarSizeInBits()*2; |
--- |
9716 |
unsigned NumBits = VT.getScalarSizeInBits()*2; |
--- |
| 9717 |
if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) |
--- |
9717 |
if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) |
--- |
| 9718 |
return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); |
--- |
9718 |
return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); |
--- |
| 9719 |
} |
--- |
9719 |
} |
--- |
| 9720 |
break; |
--- |
9720 |
break; |
--- |
| 9721 |
#endif |
--- |
9721 |
#endif |
--- |
| 9722 |
} |
--- |
9722 |
} |
--- |
| 9723 |
|
--- |
9723 |
|
--- |
| 9724 |
// Memoize the node unless it returns a flag. |
--- |
9724 |
// Memoize the node unless it returns a flag. |
--- |
| 9725 |
SDNode *N; |
--- |
9725 |
SDNode *N; |
--- |
| 9726 |
if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { |
3 |
9726 |
if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { |
3 |
| 9727 |
FoldingSetNodeID ID; |
2 |
9727 |
FoldingSetNodeID ID; |
2 |
| 9728 |
AddNodeIDNode(ID, Opcode, VTList, Ops); |
2 |
9728 |
AddNodeIDNode(ID, Opcode, VTList, Ops); |
2 |
| 9729 |
void *IP = nullptr; |
2 |
9729 |
void *IP = nullptr; |
2 |
| 9730 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) |
2 |
9730 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) |
2 |
| 9731 |
return SDValue(E, 0); |
0 |
9731 |
return SDValue(E, 0); |
0 |
| 9732 |
|
--- |
9732 |
|
--- |
| 9733 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList); |
2 |
9733 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList); |
2 |
| 9734 |
createOperands(N, Ops); |
2 |
9734 |
createOperands(N, Ops); |
2 |
| 9735 |
CSEMap.InsertNode(N, IP); |
2 |
9735 |
CSEMap.InsertNode(N, IP); |
2 |
| 9736 |
} else { |
2 |
9736 |
} else { |
2 |
| 9737 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList); |
1 |
9737 |
N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList); |
1 |
| 9738 |
createOperands(N, Ops); |
1 |
9738 |
createOperands(N, Ops); |
1 |
| 9739 |
} |
--- |
9739 |
} |
--- |
| 9740 |
|
--- |
9740 |
|
--- |
| 9741 |
N->setFlags(Flags); |
3 |
9741 |
N->setFlags(Flags); |
3 |
| 9742 |
InsertNode(N); |
3 |
9742 |
InsertNode(N); |
3 |
| 9743 |
SDValue V(N, 0); |
3 |
9743 |
SDValue V(N, 0); |
3 |
| 9744 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
3 |
9744 |
NewSDValueDbgMsg(V, "Creating new node: ", this); |
3 |
| 9745 |
return V; |
3 |
9745 |
return V; |
3 |
| 9746 |
} |
--- |
9746 |
} |
--- |
| 9747 |
|
--- |
9747 |
|
--- |
| 9748 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, |
0 |
9748 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, |
0 |
| 9749 |
SDVTList VTList) { |
--- |
9749 |
SDVTList VTList) { |
--- |
| 9750 |
return getNode(Opcode, DL, VTList, std::nullopt); |
0 |
9750 |
return getNode(Opcode, DL, VTList, std::nullopt); |
0 |
| 9751 |
} |
--- |
9751 |
} |
--- |
| 9752 |
|
--- |
9752 |
|
--- |
| 9753 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
0 |
9753 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
0 |
| 9754 |
SDValue N1) { |
--- |
9754 |
SDValue N1) { |
--- |
| 9755 |
SDValue Ops[] = { N1 }; |
0 |
9755 |
SDValue Ops[] = { N1 }; |
0 |
| 9756 |
return getNode(Opcode, DL, VTList, Ops); |
0 |
9756 |
return getNode(Opcode, DL, VTList, Ops); |
0 |
| 9757 |
} |
--- |
9757 |
} |
--- |
| 9758 |
|
--- |
9758 |
|
--- |
| 9759 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
0 |
9759 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
0 |
| 9760 |
SDValue N1, SDValue N2) { |
--- |
9760 |
SDValue N1, SDValue N2) { |
--- |
| 9761 |
SDValue Ops[] = { N1, N2 }; |
0 |
9761 |
SDValue Ops[] = { N1, N2 }; |
0 |
| 9762 |
return getNode(Opcode, DL, VTList, Ops); |
0 |
9762 |
return getNode(Opcode, DL, VTList, Ops); |
0 |
| 9763 |
} |
--- |
9763 |
} |
--- |
| 9764 |
|
--- |
9764 |
|
--- |
| 9765 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
0 |
9765 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
0 |
| 9766 |
SDValue N1, SDValue N2, SDValue N3) { |
--- |
9766 |
SDValue N1, SDValue N2, SDValue N3) { |
--- |
| 9767 |
SDValue Ops[] = { N1, N2, N3 }; |
0 |
9767 |
SDValue Ops[] = { N1, N2, N3 }; |
0 |
| 9768 |
return getNode(Opcode, DL, VTList, Ops); |
0 |
9768 |
return getNode(Opcode, DL, VTList, Ops); |
0 |
| 9769 |
} |
--- |
9769 |
} |
--- |
| 9770 |
|
--- |
9770 |
|
--- |
| 9771 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
0 |
9771 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
0 |
| 9772 |
SDValue N1, SDValue N2, SDValue N3, SDValue N4) { |
--- |
9772 |
SDValue N1, SDValue N2, SDValue N3, SDValue N4) { |
--- |
| 9773 |
SDValue Ops[] = { N1, N2, N3, N4 }; |
0 |
9773 |
SDValue Ops[] = { N1, N2, N3, N4 }; |
0 |
| 9774 |
return getNode(Opcode, DL, VTList, Ops); |
0 |
9774 |
return getNode(Opcode, DL, VTList, Ops); |
0 |
| 9775 |
} |
--- |
9775 |
} |
--- |
| 9776 |
|
--- |
9776 |
|
--- |
| 9777 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
0 |
9777 |
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
0 |
| 9778 |
SDValue N1, SDValue N2, SDValue N3, SDValue N4, |
--- |
9778 |
SDValue N1, SDValue N2, SDValue N3, SDValue N4, |
--- |
| 9779 |
SDValue N5) { |
--- |
9779 |
SDValue N5) { |
--- |
| 9780 |
SDValue Ops[] = { N1, N2, N3, N4, N5 }; |
0 |
9780 |
SDValue Ops[] = { N1, N2, N3, N4, N5 }; |
0 |
| 9781 |
return getNode(Opcode, DL, VTList, Ops); |
0 |
9781 |
return getNode(Opcode, DL, VTList, Ops); |
0 |
| 9782 |
} |
--- |
9782 |
} |
--- |
| 9783 |
|
--- |
9783 |
|
--- |
| 9784 |
SDVTList SelectionDAG::getVTList(EVT VT) { |
87 |
9784 |
SDVTList SelectionDAG::getVTList(EVT VT) { |
87 |
| 9785 |
return makeVTList(SDNode::getValueTypeList(VT), 1); |
87 |
9785 |
return makeVTList(SDNode::getValueTypeList(VT), 1); |
87 |
| 9786 |
} |
--- |
9786 |
} |
--- |
| 9787 |
|
--- |
9787 |
|
--- |
| 9788 |
SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { |
15 |
9788 |
SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { |
15 |
| 9789 |
FoldingSetNodeID ID; |
15 |
9789 |
FoldingSetNodeID ID; |
15 |
| 9790 |
ID.AddInteger(2U); |
15 |
9790 |
ID.AddInteger(2U); |
15 |
| 9791 |
ID.AddInteger(VT1.getRawBits()); |
15 |
9791 |
ID.AddInteger(VT1.getRawBits()); |
15 |
| 9792 |
ID.AddInteger(VT2.getRawBits()); |
15 |
9792 |
ID.AddInteger(VT2.getRawBits()); |
15 |
| 9793 |
|
--- |
9793 |
|
--- |
| 9794 |
void *IP = nullptr; |
15 |
9794 |
void *IP = nullptr; |
15 |
| 9795 |
SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); |
15 |
9795 |
SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); |
15 |
| 9796 |
if (!Result) { |
15 |
9796 |
if (!Result) { |
15 |
| 9797 |
EVT *Array = Allocator.Allocate(2); |
3 |
9797 |
EVT *Array = Allocator.Allocate(2); |
3 |
| 9798 |
Array[0] = VT1; |
3 |
9798 |
Array[0] = VT1; |
3 |
| 9799 |
Array[1] = VT2; |
3 |
9799 |
Array[1] = VT2; |
3 |
| 9800 |
Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2); |
3 |
9800 |
Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2); |
3 |
| 9801 |
VTListMap.InsertNode(Result, IP); |
3 |
9801 |
VTListMap.InsertNode(Result, IP); |
3 |
| 9802 |
} |
--- |
9802 |
} |
--- |
| 9803 |
return Result->getSDVTList(); |
15 |
9803 |
return Result->getSDVTList(); |
15 |
| 9804 |
} |
15 |
9804 |
} |
15 |
| 9805 |
|
--- |
9805 |
|
--- |
| 9806 |
SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { |
0 |
9806 |
SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { |
0 |
| 9807 |
FoldingSetNodeID ID; |
0 |
9807 |
FoldingSetNodeID ID; |
0 |
| 9808 |
ID.AddInteger(3U); |
0 |
9808 |
ID.AddInteger(3U); |
0 |
| 9809 |
ID.AddInteger(VT1.getRawBits()); |
0 |
9809 |
ID.AddInteger(VT1.getRawBits()); |
0 |
| 9810 |
ID.AddInteger(VT2.getRawBits()); |
0 |
9810 |
ID.AddInteger(VT2.getRawBits()); |
0 |
| 9811 |
ID.AddInteger(VT3.getRawBits()); |
0 |
9811 |
ID.AddInteger(VT3.getRawBits()); |
0 |
| 9812 |
|
--- |
9812 |
|
--- |
| 9813 |
void *IP = nullptr; |
0 |
9813 |
void *IP = nullptr; |
0 |
| 9814 |
SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); |
0 |
9814 |
SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); |
0 |
| 9815 |
if (!Result) { |
0 |
9815 |
if (!Result) { |
0 |
| 9816 |
EVT *Array = Allocator.Allocate(3); |
0 |
9816 |
EVT *Array = Allocator.Allocate(3); |
0 |
| 9817 |
Array[0] = VT1; |
0 |
9817 |
Array[0] = VT1; |
0 |
| 9818 |
Array[1] = VT2; |
0 |
9818 |
Array[1] = VT2; |
0 |
| 9819 |
Array[2] = VT3; |
0 |
9819 |
Array[2] = VT3; |
0 |
| 9820 |
Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3); |
0 |
9820 |
Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3); |
0 |
| 9821 |
VTListMap.InsertNode(Result, IP); |
0 |
9821 |
VTListMap.InsertNode(Result, IP); |
0 |
| 9822 |
} |
--- |
9822 |
} |
--- |
| 9823 |
return Result->getSDVTList(); |
0 |
9823 |
return Result->getSDVTList(); |
0 |
| 9824 |
} |
0 |
9824 |
} |
0 |
| 9825 |
|
--- |
9825 |
|
--- |
| 9826 |
SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { |
0 |
9826 |
SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { |
0 |
| 9827 |
FoldingSetNodeID ID; |
0 |
9827 |
FoldingSetNodeID ID; |
0 |
| 9828 |
ID.AddInteger(4U); |
0 |
9828 |
ID.AddInteger(4U); |
0 |
| 9829 |
ID.AddInteger(VT1.getRawBits()); |
0 |
9829 |
ID.AddInteger(VT1.getRawBits()); |
0 |
| 9830 |
ID.AddInteger(VT2.getRawBits()); |
0 |
9830 |
ID.AddInteger(VT2.getRawBits()); |
0 |
| 9831 |
ID.AddInteger(VT3.getRawBits()); |
0 |
9831 |
ID.AddInteger(VT3.getRawBits()); |
0 |
| 9832 |
ID.AddInteger(VT4.getRawBits()); |
0 |
9832 |
ID.AddInteger(VT4.getRawBits()); |
0 |
| 9833 |
|
--- |
9833 |
|
--- |
| 9834 |
void *IP = nullptr; |
0 |
9834 |
void *IP = nullptr; |
0 |
| 9835 |
SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); |
0 |
9835 |
SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); |
0 |
| 9836 |
if (!Result) { |
0 |
9836 |
if (!Result) { |
0 |
| 9837 |
EVT *Array = Allocator.Allocate(4); |
0 |
9837 |
EVT *Array = Allocator.Allocate(4); |
0 |
| 9838 |
Array[0] = VT1; |
0 |
9838 |
Array[0] = VT1; |
0 |
| 9839 |
Array[1] = VT2; |
0 |
9839 |
Array[1] = VT2; |
0 |
| 9840 |
Array[2] = VT3; |
0 |
9840 |
Array[2] = VT3; |
0 |
| 9841 |
Array[3] = VT4; |
0 |
9841 |
Array[3] = VT4; |
0 |
| 9842 |
Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4); |
0 |
9842 |
Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4); |
0 |
| 9843 |
VTListMap.InsertNode(Result, IP); |
0 |
9843 |
VTListMap.InsertNode(Result, IP); |
0 |
| 9844 |
} |
--- |
9844 |
} |
--- |
| 9845 |
return Result->getSDVTList(); |
0 |
9845 |
return Result->getSDVTList(); |
0 |
| 9846 |
} |
0 |
9846 |
} |
0 |
| 9847 |
|
--- |
9847 |
|
--- |
| 9848 |
SDVTList SelectionDAG::getVTList(ArrayRef VTs) { |
5 |
9848 |
SDVTList SelectionDAG::getVTList(ArrayRef VTs) { |
5 |
| 9849 |
unsigned NumVTs = VTs.size(); |
5 |
9849 |
unsigned NumVTs = VTs.size(); |
5 |
| 9850 |
FoldingSetNodeID ID; |
5 |
9850 |
FoldingSetNodeID ID; |
5 |
| 9851 |
ID.AddInteger(NumVTs); |
5 |
9851 |
ID.AddInteger(NumVTs); |
5 |
| 9852 |
for (unsigned index = 0; index < NumVTs; index++) { |
10 |
9852 |
for (unsigned index = 0; index < NumVTs; index++) { |
10 |
| 9853 |
ID.AddInteger(VTs[index].getRawBits()); |
5 |
9853 |
ID.AddInteger(VTs[index].getRawBits()); |
5 |
| 9854 |
} |
--- |
9854 |
} |
--- |
| 9855 |
|
--- |
9855 |
|
--- |
| 9856 |
void *IP = nullptr; |
5 |
9856 |
void *IP = nullptr; |
5 |
| 9857 |
SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); |
5 |
9857 |
SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); |
5 |
| 9858 |
if (!Result) { |
5 |
9858 |
if (!Result) { |
5 |
| 9859 |
EVT *Array = Allocator.Allocate(NumVTs); |
1 |
9859 |
EVT *Array = Allocator.Allocate(NumVTs); |
1 |
| 9860 |
llvm::copy(VTs, Array); |
1 |
9860 |
llvm::copy(VTs, Array); |
1 |
| 9861 |
Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs); |
1 |
9861 |
Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs); |
1 |
| 9862 |
VTListMap.InsertNode(Result, IP); |
1 |
9862 |
VTListMap.InsertNode(Result, IP); |
1 |
| 9863 |
} |
--- |
9863 |
} |
--- |
| 9864 |
return Result->getSDVTList(); |
5 |
9864 |
return Result->getSDVTList(); |
5 |
| 9865 |
} |
5 |
9865 |
} |
5 |
| 9866 |
|
--- |
9866 |
|
--- |
| 9867 |
|
--- |
9867 |
|
--- |
| 9868 |
/// UpdateNodeOperands - *Mutate* the specified node in-place to have the |
--- |
9868 |
/// UpdateNodeOperands - *Mutate* the specified node in-place to have the |
--- |
| 9869 |
/// specified operands. If the resultant node already exists in the DAG, |
--- |
9869 |
/// specified operands. If the resultant node already exists in the DAG, |
--- |
| 9870 |
/// this does not modify the specified node, instead it returns the node that |
--- |
9870 |
/// this does not modify the specified node, instead it returns the node that |
--- |
| 9871 |
/// already exists. If the resultant node does not exist in the DAG, the |
--- |
9871 |
/// already exists. If the resultant node does not exist in the DAG, the |
--- |
| 9872 |
/// input node is returned. As a degenerate case, if you specify the same |
--- |
9872 |
/// input node is returned. As a degenerate case, if you specify the same |
--- |
| 9873 |
/// input operands as the node already has, the input node is returned. |
--- |
9873 |
/// input operands as the node already has, the input node is returned. |
--- |
| 9874 |
SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { |
0 |
9874 |
SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { |
0 |
| 9875 |
assert(N->getNumOperands() == 1 && "Update with wrong number of operands"); |
0 |
9875 |
assert(N->getNumOperands() == 1 && "Update with wrong number of operands"); |
0 |
| 9876 |
|
--- |
9876 |
|
--- |
| 9877 |
// Check to see if there is no change. |
--- |
9877 |
// Check to see if there is no change. |
--- |
| 9878 |
if (Op == N->getOperand(0)) return N; |
0 |
9878 |
if (Op == N->getOperand(0)) return N; |
0 |
| 9879 |
|
--- |
9879 |
|
--- |
| 9880 |
// See if the modified node already exists. |
--- |
9880 |
// See if the modified node already exists. |
--- |
| 9881 |
void *InsertPos = nullptr; |
0 |
9881 |
void *InsertPos = nullptr; |
0 |
| 9882 |
if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) |
0 |
9882 |
if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) |
0 |
| 9883 |
return Existing; |
0 |
9883 |
return Existing; |
0 |
| 9884 |
|
--- |
9884 |
|
--- |
| 9885 |
// Nope it doesn't. Remove the node from its current place in the maps. |
--- |
9885 |
// Nope it doesn't. Remove the node from its current place in the maps. |
--- |
| 9886 |
if (InsertPos) |
0 |
9886 |
if (InsertPos) |
0 |
| 9887 |
if (!RemoveNodeFromCSEMaps(N)) |
0 |
9887 |
if (!RemoveNodeFromCSEMaps(N)) |
0 |
| 9888 |
InsertPos = nullptr; |
0 |
9888 |
InsertPos = nullptr; |
0 |
| 9889 |
|
--- |
9889 |
|
--- |
| 9890 |
// Now we update the operands. |
--- |
9890 |
// Now we update the operands. |
--- |
| 9891 |
N->OperandList[0].set(Op); |
0 |
9891 |
N->OperandList[0].set(Op); |
0 |
| 9892 |
|
--- |
9892 |
|
--- |
| 9893 |
updateDivergence(N); |
0 |
9893 |
updateDivergence(N); |
0 |
| 9894 |
// If this gets put into a CSE map, add it. |
--- |
9894 |
// If this gets put into a CSE map, add it. |
--- |
| 9895 |
if (InsertPos) CSEMap.InsertNode(N, InsertPos); |
0 |
9895 |
if (InsertPos) CSEMap.InsertNode(N, InsertPos); |
0 |
| 9896 |
return N; |
0 |
9896 |
return N; |
0 |
| 9897 |
} |
--- |
9897 |
} |
--- |
| 9898 |
|
--- |
9898 |
|
--- |
| 9899 |
SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { |
0 |
9899 |
SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { |
0 |
| 9900 |
assert(N->getNumOperands() == 2 && "Update with wrong number of operands"); |
0 |
9900 |
assert(N->getNumOperands() == 2 && "Update with wrong number of operands"); |
0 |
| 9901 |
|
--- |
9901 |
|
--- |
| 9902 |
// Check to see if there is no change. |
--- |
9902 |
// Check to see if there is no change. |
--- |
| 9903 |
if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1)) |
0 |
9903 |
if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1)) |
0 |
| 9904 |
return N; // No operands changed, just return the input node. |
0 |
9904 |
return N; // No operands changed, just return the input node. |
0 |
| 9905 |
|
--- |
9905 |
|
--- |
| 9906 |
// See if the modified node already exists. |
--- |
9906 |
// See if the modified node already exists. |
--- |
| 9907 |
void *InsertPos = nullptr; |
0 |
9907 |
void *InsertPos = nullptr; |
0 |
| 9908 |
if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) |
0 |
9908 |
if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) |
0 |
| 9909 |
return Existing; |
0 |
9909 |
return Existing; |
0 |
| 9910 |
|
--- |
9910 |
|
--- |
| 9911 |
// Nope it doesn't. Remove the node from its current place in the maps. |
--- |
9911 |
// Nope it doesn't. Remove the node from its current place in the maps. |
--- |
| 9912 |
if (InsertPos) |
0 |
9912 |
if (InsertPos) |
0 |
| 9913 |
if (!RemoveNodeFromCSEMaps(N)) |
0 |
9913 |
if (!RemoveNodeFromCSEMaps(N)) |
0 |
| 9914 |
InsertPos = nullptr; |
0 |
9914 |
InsertPos = nullptr; |
0 |
| 9915 |
|
--- |
9915 |
|
--- |
| 9916 |
// Now we update the operands. |
--- |
9916 |
// Now we update the operands. |
--- |
| 9917 |
if (N->OperandList[0] != Op1) |
0 |
9917 |
if (N->OperandList[0] != Op1) |
0 |
| 9918 |
N->OperandList[0].set(Op1); |
0 |
9918 |
N->OperandList[0].set(Op1); |
0 |
| 9919 |
if (N->OperandList[1] != Op2) |
0 |
9919 |
if (N->OperandList[1] != Op2) |
0 |
| 9920 |
N->OperandList[1].set(Op2); |
0 |
9920 |
N->OperandList[1].set(Op2); |
0 |
| 9921 |
|
--- |
9921 |
|
--- |
| 9922 |
updateDivergence(N); |
0 |
9922 |
updateDivergence(N); |
0 |
| 9923 |
// If this gets put into a CSE map, add it. |
--- |
9923 |
// If this gets put into a CSE map, add it. |
--- |
| 9924 |
if (InsertPos) CSEMap.InsertNode(N, InsertPos); |
0 |
9924 |
if (InsertPos) CSEMap.InsertNode(N, InsertPos); |
0 |
| 9925 |
return N; |
0 |
9925 |
return N; |
0 |
| 9926 |
} |
--- |
9926 |
} |
--- |
| 9927 |
|
--- |
9927 |
|
--- |
| 9928 |
SDNode *SelectionDAG:: |
1 |
9928 |
SDNode *SelectionDAG:: |
1 |
| 9929 |
UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { |
--- |
9929 |
UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { |
--- |
| 9930 |
SDValue Ops[] = { Op1, Op2, Op3 }; |
1 |
9930 |
SDValue Ops[] = { Op1, Op2, Op3 }; |
1 |
| 9931 |
return UpdateNodeOperands(N, Ops); |
1 |
9931 |
return UpdateNodeOperands(N, Ops); |
1 |
| 9932 |
} |
--- |
9932 |
} |
--- |
| 9933 |
|
--- |
9933 |
|
--- |
| 9934 |
SDNode *SelectionDAG:: |
0 |
9934 |
SDNode *SelectionDAG:: |
0 |
| 9935 |
UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, |
--- |
9935 |
UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, |
--- |
| 9936 |
SDValue Op3, SDValue Op4) { |
--- |
9936 |
SDValue Op3, SDValue Op4) { |
--- |
| 9937 |
SDValue Ops[] = { Op1, Op2, Op3, Op4 }; |
0 |
9937 |
SDValue Ops[] = { Op1, Op2, Op3, Op4 }; |
0 |
| 9938 |
return UpdateNodeOperands(N, Ops); |
0 |
9938 |
return UpdateNodeOperands(N, Ops); |
0 |
| 9939 |
} |
--- |
9939 |
} |
--- |
| 9940 |
|
--- |
9940 |
|
--- |
| 9941 |
SDNode *SelectionDAG:: |
0 |
9941 |
SDNode *SelectionDAG:: |
0 |
| 9942 |
UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, |
--- |
9942 |
UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, |
--- |
| 9943 |
SDValue Op3, SDValue Op4, SDValue Op5) { |
--- |
9943 |
SDValue Op3, SDValue Op4, SDValue Op5) { |
--- |
| 9944 |
SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; |
0 |
9944 |
SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; |
0 |
| 9945 |
return UpdateNodeOperands(N, Ops); |
0 |
9945 |
return UpdateNodeOperands(N, Ops); |
0 |
| 9946 |
} |
--- |
9946 |
} |
--- |
| 9947 |
|
--- |
9947 |
|
--- |
| 9948 |
SDNode *SelectionDAG:: |
1 |
9948 |
SDNode *SelectionDAG:: |
1 |
| 9949 |
UpdateNodeOperands(SDNode *N, ArrayRef Ops) { |
--- |
9949 |
UpdateNodeOperands(SDNode *N, ArrayRef Ops) { |
--- |
| 9950 |
unsigned NumOps = Ops.size(); |
1 |
9950 |
unsigned NumOps = Ops.size(); |
1 |
| 9951 |
assert(N->getNumOperands() == NumOps && |
1 |
9951 |
assert(N->getNumOperands() == NumOps && |
1 |
| 9952 |
"Update with wrong number of operands"); |
--- |
9952 |
"Update with wrong number of operands"); |
--- |
| 9953 |
|
--- |
9953 |
|
--- |
| 9954 |
// If no operands changed just return the input node. |
--- |
9954 |
// If no operands changed just return the input node. |
--- |
| 9955 |
if (std::equal(Ops.begin(), Ops.end(), N->op_begin())) |
1 |
9955 |
if (std::equal(Ops.begin(), Ops.end(), N->op_begin())) |
1 |
| 9956 |
return N; |
0 |
9956 |
return N; |
0 |
| 9957 |
|
--- |
9957 |
|
--- |
| 9958 |
// See if the modified node already exists. |
--- |
9958 |
// See if the modified node already exists. |
--- |
| 9959 |
void *InsertPos = nullptr; |
1 |
9959 |
void *InsertPos = nullptr; |
1 |
| 9960 |
if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos)) |
1 |
9960 |
if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos)) |
1 |
| 9961 |
return Existing; |
0 |
9961 |
return Existing; |
0 |
| 9962 |
|
--- |
9962 |
|
--- |
| 9963 |
// Nope it doesn't. Remove the node from its current place in the maps. |
--- |
9963 |
// Nope it doesn't. Remove the node from its current place in the maps. |
--- |
| 9964 |
if (InsertPos) |
1 |
9964 |
if (InsertPos) |
1 |
| 9965 |
if (!RemoveNodeFromCSEMaps(N)) |
1 |
9965 |
if (!RemoveNodeFromCSEMaps(N)) |
1 |
| 9966 |
InsertPos = nullptr; |
0 |
9966 |
InsertPos = nullptr; |
0 |
| 9967 |
|
--- |
9967 |
|
--- |
| 9968 |
// Now we update the operands. |
--- |
9968 |
// Now we update the operands. |
--- |
| 9969 |
for (unsigned i = 0; i != NumOps; ++i) |
4 |
9969 |
for (unsigned i = 0; i != NumOps; ++i) |
4 |
| 9970 |
if (N->OperandList[i] != Ops[i]) |
3 |
9970 |
if (N->OperandList[i] != Ops[i]) |
3 |
| 9971 |
N->OperandList[i].set(Ops[i]); |
1 |
9971 |
N->OperandList[i].set(Ops[i]); |
1 |
| 9972 |
|
--- |
9972 |
|
--- |
| 9973 |
updateDivergence(N); |
1 |
9973 |
updateDivergence(N); |
1 |
| 9974 |
// If this gets put into a CSE map, add it. |
--- |
9974 |
// If this gets put into a CSE map, add it. |
--- |
| 9975 |
if (InsertPos) CSEMap.InsertNode(N, InsertPos); |
1 |
9975 |
if (InsertPos) CSEMap.InsertNode(N, InsertPos); |
1 |
| 9976 |
return N; |
1 |
9976 |
return N; |
1 |
| 9977 |
} |
--- |
9977 |
} |
--- |
| 9978 |
|
--- |
9978 |
|
--- |
| 9979 |
/// DropOperands - Release the operands and set this node to have |
--- |
9979 |
/// DropOperands - Release the operands and set this node to have |
--- |
| 9980 |
/// zero operands. |
--- |
9980 |
/// zero operands. |
--- |
| 9981 |
void SDNode::DropOperands() { |
50 |
9981 |
void SDNode::DropOperands() { |
50 |
| 9982 |
// Unlike the code in MorphNodeTo that does this, we don't need to |
--- |
9982 |
// Unlike the code in MorphNodeTo that does this, we don't need to |
--- |
| 9983 |
// watch for dead nodes here. |
--- |
9983 |
// watch for dead nodes here. |
--- |
| 9984 |
for (op_iterator I = op_begin(), E = op_end(); I != E; ) { |
99 |
9984 |
for (op_iterator I = op_begin(), E = op_end(); I != E; ) { |
99 |
| 9985 |
SDUse &Use = *I++; |
49 |
9985 |
SDUse &Use = *I++; |
49 |
| 9986 |
Use.set(SDValue()); |
49 |
9986 |
Use.set(SDValue()); |
49 |
| 9987 |
} |
--- |
9987 |
} |
--- |
| 9988 |
} |
50 |
9988 |
} |
50 |
| 9989 |
|
--- |
9989 |
|
--- |
| 9990 |
void SelectionDAG::setNodeMemRefs(MachineSDNode *N, |
9 |
9990 |
void SelectionDAG::setNodeMemRefs(MachineSDNode *N, |
9 |
| 9991 |
ArrayRef NewMemRefs) { |
--- |
9991 |
ArrayRef NewMemRefs) { |
--- |
| 9992 |
if (NewMemRefs.empty()) { |
9 |
9992 |
if (NewMemRefs.empty()) { |
9 |
| 9993 |
N->clearMemRefs(); |
0 |
9993 |
N->clearMemRefs(); |
0 |
| 9994 |
return; |
0 |
9994 |
return; |
0 |
| 9995 |
} |
--- |
9995 |
} |
--- |
| 9996 |
|
--- |
9996 |
|
--- |
| 9997 |
// Check if we can avoid allocating by storing a single reference directly. |
--- |
9997 |
// Check if we can avoid allocating by storing a single reference directly. |
--- |
| 9998 |
if (NewMemRefs.size() == 1) { |
9 |
9998 |
if (NewMemRefs.size() == 1) { |
9 |
| 9999 |
N->MemRefs = NewMemRefs[0]; |
9 |
9999 |
N->MemRefs = NewMemRefs[0]; |
9 |
| 10000 |
N->NumMemRefs = 1; |
9 |
10000 |
N->NumMemRefs = 1; |
9 |
| 10001 |
return; |
9 |
10001 |
return; |
9 |
| 10002 |
} |
--- |
10002 |
} |
--- |
| 10003 |
|
--- |
10003 |
|
--- |
| 10004 |
MachineMemOperand **MemRefsBuffer = |
--- |
10004 |
MachineMemOperand **MemRefsBuffer = |
--- |
| 10005 |
Allocator.template Allocate(NewMemRefs.size()); |
0 |
10005 |
Allocator.template Allocate(NewMemRefs.size()); |
0 |
| 10006 |
llvm::copy(NewMemRefs, MemRefsBuffer); |
0 |
10006 |
llvm::copy(NewMemRefs, MemRefsBuffer); |
0 |
| 10007 |
N->MemRefs = MemRefsBuffer; |
0 |
10007 |
N->MemRefs = MemRefsBuffer; |
0 |
| 10008 |
N->NumMemRefs = static_cast(NewMemRefs.size()); |
0 |
10008 |
N->NumMemRefs = static_cast(NewMemRefs.size()); |
0 |
| 10009 |
} |
--- |
10009 |
} |
--- |
| 10010 |
|
--- |
10010 |
|
--- |
| 10011 |
/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a |
--- |
10011 |
/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a |
--- |
| 10012 |
/// machine opcode. |
--- |
10012 |
/// machine opcode. |
--- |
| 10013 |
/// |
--- |
10013 |
/// |
--- |
| 10014 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
10014 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
| 10015 |
EVT VT) { |
--- |
10015 |
EVT VT) { |
--- |
| 10016 |
SDVTList VTs = getVTList(VT); |
0 |
10016 |
SDVTList VTs = getVTList(VT); |
0 |
| 10017 |
return SelectNodeTo(N, MachineOpc, VTs, std::nullopt); |
0 |
10017 |
return SelectNodeTo(N, MachineOpc, VTs, std::nullopt); |
0 |
| 10018 |
} |
--- |
10018 |
} |
--- |
| 10019 |
|
--- |
10019 |
|
--- |
| 10020 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
10020 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
| 10021 |
EVT VT, SDValue Op1) { |
--- |
10021 |
EVT VT, SDValue Op1) { |
--- |
| 10022 |
SDVTList VTs = getVTList(VT); |
0 |
10022 |
SDVTList VTs = getVTList(VT); |
0 |
| 10023 |
SDValue Ops[] = { Op1 }; |
0 |
10023 |
SDValue Ops[] = { Op1 }; |
0 |
| 10024 |
return SelectNodeTo(N, MachineOpc, VTs, Ops); |
0 |
10024 |
return SelectNodeTo(N, MachineOpc, VTs, Ops); |
0 |
| 10025 |
} |
--- |
10025 |
} |
--- |
| 10026 |
|
--- |
10026 |
|
--- |
| 10027 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
10027 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
| 10028 |
EVT VT, SDValue Op1, |
--- |
10028 |
EVT VT, SDValue Op1, |
--- |
| 10029 |
SDValue Op2) { |
--- |
10029 |
SDValue Op2) { |
--- |
| 10030 |
SDVTList VTs = getVTList(VT); |
0 |
10030 |
SDVTList VTs = getVTList(VT); |
0 |
| 10031 |
SDValue Ops[] = { Op1, Op2 }; |
0 |
10031 |
SDValue Ops[] = { Op1, Op2 }; |
0 |
| 10032 |
return SelectNodeTo(N, MachineOpc, VTs, Ops); |
0 |
10032 |
return SelectNodeTo(N, MachineOpc, VTs, Ops); |
0 |
| 10033 |
} |
--- |
10033 |
} |
--- |
| 10034 |
|
--- |
10034 |
|
--- |
| 10035 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
10035 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
| 10036 |
EVT VT, SDValue Op1, |
--- |
10036 |
EVT VT, SDValue Op1, |
--- |
| 10037 |
SDValue Op2, SDValue Op3) { |
--- |
10037 |
SDValue Op2, SDValue Op3) { |
--- |
| 10038 |
SDVTList VTs = getVTList(VT); |
0 |
10038 |
SDVTList VTs = getVTList(VT); |
0 |
| 10039 |
SDValue Ops[] = { Op1, Op2, Op3 }; |
0 |
10039 |
SDValue Ops[] = { Op1, Op2, Op3 }; |
0 |
| 10040 |
return SelectNodeTo(N, MachineOpc, VTs, Ops); |
0 |
10040 |
return SelectNodeTo(N, MachineOpc, VTs, Ops); |
0 |
| 10041 |
} |
--- |
10041 |
} |
--- |
| 10042 |
|
--- |
10042 |
|
--- |
| 10043 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
10043 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
| 10044 |
EVT VT, ArrayRef Ops) { |
--- |
10044 |
EVT VT, ArrayRef Ops) { |
--- |
| 10045 |
SDVTList VTs = getVTList(VT); |
0 |
10045 |
SDVTList VTs = getVTList(VT); |
0 |
| 10046 |
return SelectNodeTo(N, MachineOpc, VTs, Ops); |
0 |
10046 |
return SelectNodeTo(N, MachineOpc, VTs, Ops); |
0 |
| 10047 |
} |
--- |
10047 |
} |
--- |
| 10048 |
|
--- |
10048 |
|
--- |
| 10049 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
10049 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
| 10050 |
EVT VT1, EVT VT2, ArrayRef Ops) { |
--- |
10050 |
EVT VT1, EVT VT2, ArrayRef Ops) { |
--- |
| 10051 |
SDVTList VTs = getVTList(VT1, VT2); |
0 |
10051 |
SDVTList VTs = getVTList(VT1, VT2); |
0 |
| 10052 |
return SelectNodeTo(N, MachineOpc, VTs, Ops); |
0 |
10052 |
return SelectNodeTo(N, MachineOpc, VTs, Ops); |
0 |
| 10053 |
} |
--- |
10053 |
} |
--- |
| 10054 |
|
--- |
10054 |
|
--- |
| 10055 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
10055 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
| 10056 |
EVT VT1, EVT VT2) { |
--- |
10056 |
EVT VT1, EVT VT2) { |
--- |
| 10057 |
SDVTList VTs = getVTList(VT1, VT2); |
0 |
10057 |
SDVTList VTs = getVTList(VT1, VT2); |
0 |
| 10058 |
return SelectNodeTo(N, MachineOpc, VTs, std::nullopt); |
0 |
10058 |
return SelectNodeTo(N, MachineOpc, VTs, std::nullopt); |
0 |
| 10059 |
} |
--- |
10059 |
} |
--- |
| 10060 |
|
--- |
10060 |
|
--- |
| 10061 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
10061 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
| 10062 |
EVT VT1, EVT VT2, EVT VT3, |
--- |
10062 |
EVT VT1, EVT VT2, EVT VT3, |
--- |
| 10063 |
ArrayRef Ops) { |
--- |
10063 |
ArrayRef Ops) { |
--- |
| 10064 |
SDVTList VTs = getVTList(VT1, VT2, VT3); |
0 |
10064 |
SDVTList VTs = getVTList(VT1, VT2, VT3); |
0 |
| 10065 |
return SelectNodeTo(N, MachineOpc, VTs, Ops); |
0 |
10065 |
return SelectNodeTo(N, MachineOpc, VTs, Ops); |
0 |
| 10066 |
} |
--- |
10066 |
} |
--- |
| 10067 |
|
--- |
10067 |
|
--- |
| 10068 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
10068 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
| 10069 |
EVT VT1, EVT VT2, |
--- |
10069 |
EVT VT1, EVT VT2, |
--- |
| 10070 |
SDValue Op1, SDValue Op2) { |
--- |
10070 |
SDValue Op1, SDValue Op2) { |
--- |
| 10071 |
SDVTList VTs = getVTList(VT1, VT2); |
0 |
10071 |
SDVTList VTs = getVTList(VT1, VT2); |
0 |
| 10072 |
SDValue Ops[] = { Op1, Op2 }; |
0 |
10072 |
SDValue Ops[] = { Op1, Op2 }; |
0 |
| 10073 |
return SelectNodeTo(N, MachineOpc, VTs, Ops); |
0 |
10073 |
return SelectNodeTo(N, MachineOpc, VTs, Ops); |
0 |
| 10074 |
} |
--- |
10074 |
} |
--- |
| 10075 |
|
--- |
10075 |
|
--- |
| 10076 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
10076 |
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
0 |
| 10077 |
SDVTList VTs,ArrayRef Ops) { |
--- |
10077 |
SDVTList VTs,ArrayRef Ops) { |
--- |
| 10078 |
SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops); |
0 |
10078 |
SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops); |
0 |
| 10079 |
// Reset the NodeID to -1. |
--- |
10079 |
// Reset the NodeID to -1. |
--- |
| 10080 |
New->setNodeId(-1); |
0 |
10080 |
New->setNodeId(-1); |
0 |
| 10081 |
if (New != N) { |
0 |
10081 |
if (New != N) { |
0 |
| 10082 |
ReplaceAllUsesWith(N, New); |
0 |
10082 |
ReplaceAllUsesWith(N, New); |
0 |
| 10083 |
RemoveDeadNode(N); |
0 |
10083 |
RemoveDeadNode(N); |
0 |
| 10084 |
} |
--- |
10084 |
} |
--- |
| 10085 |
return New; |
0 |
10085 |
return New; |
0 |
| 10086 |
} |
--- |
10086 |
} |
--- |
| 10087 |
|
--- |
10087 |
|
--- |
| 10088 |
/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away |
--- |
10088 |
/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away |
--- |
| 10089 |
/// the line number information on the merged node since it is not possible to |
--- |
10089 |
/// the line number information on the merged node since it is not possible to |
--- |
| 10090 |
/// preserve the information that operation is associated with multiple lines. |
--- |
10090 |
/// preserve the information that operation is associated with multiple lines. |
--- |
| 10091 |
/// This will make the debugger working better at -O0, were there is a higher |
--- |
10091 |
/// This will make the debugger working better at -O0, were there is a higher |
--- |
| 10092 |
/// probability having other instructions associated with that line. |
--- |
10092 |
/// probability having other instructions associated with that line. |
--- |
| 10093 |
/// |
--- |
10093 |
/// |
--- |
| 10094 |
/// For IROrder, we keep the smaller of the two |
--- |
10094 |
/// For IROrder, we keep the smaller of the two |
--- |
| 10095 |
SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) { |
0 |
10095 |
SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) { |
0 |
| 10096 |
DebugLoc NLoc = N->getDebugLoc(); |
0 |
10096 |
DebugLoc NLoc = N->getDebugLoc(); |
0 |
| 10097 |
if (NLoc && OptLevel == CodeGenOpt::None && OLoc.getDebugLoc() != NLoc) { |
0 |
10097 |
if (NLoc && OptLevel == CodeGenOpt::None && OLoc.getDebugLoc() != NLoc) { |
0 |
| 10098 |
N->setDebugLoc(DebugLoc()); |
0 |
10098 |
N->setDebugLoc(DebugLoc()); |
0 |
| 10099 |
} |
--- |
10099 |
} |
--- |
| 10100 |
unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder()); |
0 |
10100 |
unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder()); |
0 |
| 10101 |
N->setIROrder(Order); |
0 |
10101 |
N->setIROrder(Order); |
0 |
| 10102 |
return N; |
0 |
10102 |
return N; |
0 |
| 10103 |
} |
0 |
10103 |
} |
0 |
| 10104 |
|
--- |
10104 |
|
--- |
| 10105 |
/// MorphNodeTo - This *mutates* the specified node to have the specified |
--- |
10105 |
/// MorphNodeTo - This *mutates* the specified node to have the specified |
--- |
| 10106 |
/// return type, opcode, and operands. |
--- |
10106 |
/// return type, opcode, and operands. |
--- |
| 10107 |
/// |
--- |
10107 |
/// |
--- |
| 10108 |
/// Note that MorphNodeTo returns the resultant node. If there is already a |
--- |
10108 |
/// Note that MorphNodeTo returns the resultant node. If there is already a |
--- |
| 10109 |
/// node of the specified opcode and operands, it returns that node instead of |
--- |
10109 |
/// node of the specified opcode and operands, it returns that node instead of |
--- |
| 10110 |
/// the current one. Note that the SDLoc need not be the same. |
--- |
10110 |
/// the current one. Note that the SDLoc need not be the same. |
--- |
| 10111 |
/// |
--- |
10111 |
/// |
--- |
| 10112 |
/// Using MorphNodeTo is faster than creating a new node and swapping it in |
--- |
10112 |
/// Using MorphNodeTo is faster than creating a new node and swapping it in |
--- |
| 10113 |
/// with ReplaceAllUsesWith both because it often avoids allocating a new |
--- |
10113 |
/// with ReplaceAllUsesWith both because it often avoids allocating a new |
--- |
| 10114 |
/// node, and because it doesn't require CSE recalculation for any of |
--- |
10114 |
/// node, and because it doesn't require CSE recalculation for any of |
--- |
| 10115 |
/// the node's users. |
--- |
10115 |
/// the node's users. |
--- |
| 10116 |
/// |
--- |
10116 |
/// |
--- |
| 10117 |
/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG. |
--- |
10117 |
/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG. |
--- |
| 10118 |
/// As a consequence it isn't appropriate to use from within the DAG combiner or |
--- |
10118 |
/// As a consequence it isn't appropriate to use from within the DAG combiner or |
--- |
| 10119 |
/// the legalizer which maintain worklists that would need to be updated when |
--- |
10119 |
/// the legalizer which maintain worklists that would need to be updated when |
--- |
| 10120 |
/// deleting things. |
--- |
10120 |
/// deleting things. |
--- |
| 10121 |
SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, |
14 |
10121 |
SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, |
14 |
| 10122 |
SDVTList VTs, ArrayRef Ops) { |
--- |
10122 |
SDVTList VTs, ArrayRef Ops) { |
--- |
| 10123 |
// If an identical node already exists, use it. |
--- |
10123 |
// If an identical node already exists, use it. |
--- |
| 10124 |
void *IP = nullptr; |
14 |
10124 |
void *IP = nullptr; |
14 |
| 10125 |
if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { |
14 |
10125 |
if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { |
14 |
| 10126 |
FoldingSetNodeID ID; |
14 |
10126 |
FoldingSetNodeID ID; |
14 |
| 10127 |
AddNodeIDNode(ID, Opc, VTs, Ops); |
14 |
10127 |
AddNodeIDNode(ID, Opc, VTs, Ops); |
14 |
| 10128 |
if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP)) |
14 |
10128 |
if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP)) |
14 |
| 10129 |
return UpdateSDLocOnMergeSDNode(ON, SDLoc(N)); |
0 |
10129 |
return UpdateSDLocOnMergeSDNode(ON, SDLoc(N)); |
0 |
| 10130 |
} |
14 |
10130 |
} |
14 |
| 10131 |
|
--- |
10131 |
|
--- |
| 10132 |
if (!RemoveNodeFromCSEMaps(N)) |
14 |
10132 |
if (!RemoveNodeFromCSEMaps(N)) |
14 |
| 10133 |
IP = nullptr; |
0 |
10133 |
IP = nullptr; |
0 |
| 10134 |
|
--- |
10134 |
|
--- |
| 10135 |
// Start the morphing. |
--- |
10135 |
// Start the morphing. |
--- |
| 10136 |
N->NodeType = Opc; |
14 |
10136 |
N->NodeType = Opc; |
14 |
| 10137 |
N->ValueList = VTs.VTs; |
14 |
10137 |
N->ValueList = VTs.VTs; |
14 |
| 10138 |
N->NumValues = VTs.NumVTs; |
14 |
10138 |
N->NumValues = VTs.NumVTs; |
14 |
| 10139 |
|
--- |
10139 |
|
--- |
| 10140 |
// Clear the operands list, updating used nodes to remove this from their |
--- |
10140 |
// Clear the operands list, updating used nodes to remove this from their |
--- |
| 10141 |
// use list. Keep track of any operands that become dead as a result. |
--- |
10141 |
// use list. Keep track of any operands that become dead as a result. |
--- |
| 10142 |
SmallPtrSet DeadNodeSet; |
14 |
10142 |
SmallPtrSet DeadNodeSet; |
14 |
| 10143 |
for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { |
57 |
10143 |
for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { |
57 |
| 10144 |
SDUse &Use = *I++; |
43 |
10144 |
SDUse &Use = *I++; |
43 |
| 10145 |
SDNode *Used = Use.getNode(); |
43 |
10145 |
SDNode *Used = Use.getNode(); |
43 |
| 10146 |
Use.set(SDValue()); |
43 |
10146 |
Use.set(SDValue()); |
43 |
| 10147 |
if (Used->use_empty()) |
43 |
10147 |
if (Used->use_empty()) |
43 |
| 10148 |
DeadNodeSet.insert(Used); |
27 |
10148 |
DeadNodeSet.insert(Used); |
27 |
| 10149 |
} |
--- |
10149 |
} |
--- |
| 10150 |
|
--- |
10150 |
|
--- |
| 10151 |
// For MachineNode, initialize the memory references information. |
--- |
10151 |
// For MachineNode, initialize the memory references information. |
--- |
| 10152 |
if (MachineSDNode *MN = dyn_cast(N)) |
14 |
10152 |
if (MachineSDNode *MN = dyn_cast(N)) |
14 |
| 10153 |
MN->clearMemRefs(); |
14 |
10153 |
MN->clearMemRefs(); |
14 |
| 10154 |
|
--- |
10154 |
|
--- |
| 10155 |
// Swap for an appropriately sized array from the recycler. |
--- |
10155 |
// Swap for an appropriately sized array from the recycler. |
--- |
| 10156 |
removeOperands(N); |
14 |
10156 |
removeOperands(N); |
14 |
| 10157 |
createOperands(N, Ops); |
14 |
10157 |
createOperands(N, Ops); |
14 |
| 10158 |
|
--- |
10158 |
|
--- |
| 10159 |
// Delete any nodes that are still dead after adding the uses for the |
--- |
10159 |
// Delete any nodes that are still dead after adding the uses for the |
--- |
| 10160 |
// new operands. |
--- |
10160 |
// new operands. |
--- |
| 10161 |
if (!DeadNodeSet.empty()) { |
14 |
10161 |
if (!DeadNodeSet.empty()) { |
14 |
| 10162 |
SmallVector DeadNodes; |
12 |
10162 |
SmallVector DeadNodes; |
12 |
| 10163 |
for (SDNode *N : DeadNodeSet) |
39 |
10163 |
for (SDNode *N : DeadNodeSet) |
39 |
| 10164 |
if (N->use_empty()) |
27 |
10164 |
if (N->use_empty()) |
27 |
| 10165 |
DeadNodes.push_back(N); |
12 |
10165 |
DeadNodes.push_back(N); |
12 |
| 10166 |
RemoveDeadNodes(DeadNodes); |
12 |
10166 |
RemoveDeadNodes(DeadNodes); |
12 |
| 10167 |
} |
12 |
10167 |
} |
12 |
| 10168 |
|
--- |
10168 |
|
--- |
| 10169 |
if (IP) |
14 |
10169 |
if (IP) |
14 |
| 10170 |
CSEMap.InsertNode(N, IP); // Memoize the new node. |
14 |
10170 |
CSEMap.InsertNode(N, IP); // Memoize the new node. |
14 |
| 10171 |
return N; |
14 |
10171 |
return N; |
14 |
| 10172 |
} |
14 |
10172 |
} |
14 |
| 10173 |
|
--- |
10173 |
|
--- |
| 10174 |
SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) { |
0 |
10174 |
SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) { |
0 |
| 10175 |
unsigned OrigOpc = Node->getOpcode(); |
0 |
10175 |
unsigned OrigOpc = Node->getOpcode(); |
0 |
| 10176 |
unsigned NewOpc; |
--- |
10176 |
unsigned NewOpc; |
--- |
| 10177 |
switch (OrigOpc) { |
0 |
10177 |
switch (OrigOpc) { |
0 |
| 10178 |
default: |
0 |
10178 |
default: |
0 |
| 10179 |
llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!"); |
0 |
10179 |
llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!"); |
0 |
| 10180 |
#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ |
--- |
10180 |
#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ |
--- |
| 10181 |
case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break; |
--- |
10181 |
case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break; |
--- |
| 10182 |
#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ |
--- |
10182 |
#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ |
--- |
| 10183 |
case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break; |
--- |
10183 |
case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break; |
--- |
| 10184 |
#include "llvm/IR/ConstrainedOps.def" |
--- |
10184 |
#include "llvm/IR/ConstrainedOps.def" |
--- |
| 10185 |
} |
--- |
10185 |
} |
--- |
| 10186 |
|
--- |
10186 |
|
--- |
| 10187 |
assert(Node->getNumValues() == 2 && "Unexpected number of results!"); |
0 |
10187 |
assert(Node->getNumValues() == 2 && "Unexpected number of results!"); |
0 |
| 10188 |
|
--- |
10188 |
|
--- |
| 10189 |
// We're taking this node out of the chain, so we need to re-link things. |
--- |
10189 |
// We're taking this node out of the chain, so we need to re-link things. |
--- |
| 10190 |
SDValue InputChain = Node->getOperand(0); |
0 |
10190 |
SDValue InputChain = Node->getOperand(0); |
0 |
| 10191 |
SDValue OutputChain = SDValue(Node, 1); |
0 |
10191 |
SDValue OutputChain = SDValue(Node, 1); |
0 |
| 10192 |
ReplaceAllUsesOfValueWith(OutputChain, InputChain); |
0 |
10192 |
ReplaceAllUsesOfValueWith(OutputChain, InputChain); |
0 |
| 10193 |
|
--- |
10193 |
|
--- |
| 10194 |
SmallVector Ops; |
0 |
10194 |
SmallVector Ops; |
0 |
| 10195 |
for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) |
0 |
10195 |
for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) |
0 |
| 10196 |
Ops.push_back(Node->getOperand(i)); |
0 |
10196 |
Ops.push_back(Node->getOperand(i)); |
0 |
| 10197 |
|
--- |
10197 |
|
--- |
| 10198 |
SDVTList VTs = getVTList(Node->getValueType(0)); |
0 |
10198 |
SDVTList VTs = getVTList(Node->getValueType(0)); |
0 |
| 10199 |
SDNode *Res = MorphNodeTo(Node, NewOpc, VTs, Ops); |
0 |
10199 |
SDNode *Res = MorphNodeTo(Node, NewOpc, VTs, Ops); |
0 |
| 10200 |
|
--- |
10200 |
|
--- |
| 10201 |
// MorphNodeTo can operate in two ways: if an existing node with the |
--- |
10201 |
// MorphNodeTo can operate in two ways: if an existing node with the |
--- |
| 10202 |
// specified operands exists, it can just return it. Otherwise, it |
--- |
10202 |
// specified operands exists, it can just return it. Otherwise, it |
--- |
| 10203 |
// updates the node in place to have the requested operands. |
--- |
10203 |
// updates the node in place to have the requested operands. |
--- |
| 10204 |
if (Res == Node) { |
0 |
10204 |
if (Res == Node) { |
0 |
| 10205 |
// If we updated the node in place, reset the node ID. To the isel, |
--- |
10205 |
// If we updated the node in place, reset the node ID. To the isel, |
--- |
| 10206 |
// this should be just like a newly allocated machine node. |
--- |
10206 |
// this should be just like a newly allocated machine node. |
--- |
| 10207 |
Res->setNodeId(-1); |
0 |
10207 |
Res->setNodeId(-1); |
0 |
| 10208 |
} else { |
--- |
10208 |
} else { |
--- |
| 10209 |
ReplaceAllUsesWith(Node, Res); |
0 |
10209 |
ReplaceAllUsesWith(Node, Res); |
0 |
| 10210 |
RemoveDeadNode(Node); |
0 |
10210 |
RemoveDeadNode(Node); |
0 |
| 10211 |
} |
--- |
10211 |
} |
--- |
| 10212 |
|
--- |
10212 |
|
--- |
| 10213 |
return Res; |
0 |
10213 |
return Res; |
0 |
| 10214 |
} |
--- |
10214 |
} |
--- |
| 10215 |
|
--- |
10215 |
|
--- |
| 10216 |
/// getMachineNode - These are used for target selectors to create a new node |
--- |
10216 |
/// getMachineNode - These are used for target selectors to create a new node |
--- |
| 10217 |
/// with specified return type(s), MachineInstr opcode, and operands. |
--- |
10217 |
/// with specified return type(s), MachineInstr opcode, and operands. |
--- |
| 10218 |
/// |
--- |
10218 |
/// |
--- |
| 10219 |
/// Note that getMachineNode returns the resultant node. If there is already a |
--- |
10219 |
/// Note that getMachineNode returns the resultant node. If there is already a |
--- |
| 10220 |
/// node of the specified opcode and operands, it returns that node instead of |
--- |
10220 |
/// node of the specified opcode and operands, it returns that node instead of |
--- |
| 10221 |
/// the current one. |
--- |
10221 |
/// the current one. |
--- |
| 10222 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
10222 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
| 10223 |
EVT VT) { |
--- |
10223 |
EVT VT) { |
--- |
| 10224 |
SDVTList VTs = getVTList(VT); |
0 |
10224 |
SDVTList VTs = getVTList(VT); |
0 |
| 10225 |
return getMachineNode(Opcode, dl, VTs, std::nullopt); |
0 |
10225 |
return getMachineNode(Opcode, dl, VTs, std::nullopt); |
0 |
| 10226 |
} |
--- |
10226 |
} |
--- |
| 10227 |
|
--- |
10227 |
|
--- |
| 10228 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
10228 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
| 10229 |
EVT VT, SDValue Op1) { |
--- |
10229 |
EVT VT, SDValue Op1) { |
--- |
| 10230 |
SDVTList VTs = getVTList(VT); |
0 |
10230 |
SDVTList VTs = getVTList(VT); |
0 |
| 10231 |
SDValue Ops[] = { Op1 }; |
0 |
10231 |
SDValue Ops[] = { Op1 }; |
0 |
| 10232 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
10232 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
| 10233 |
} |
--- |
10233 |
} |
--- |
| 10234 |
|
--- |
10234 |
|
--- |
| 10235 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
10235 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
| 10236 |
EVT VT, SDValue Op1, SDValue Op2) { |
--- |
10236 |
EVT VT, SDValue Op1, SDValue Op2) { |
--- |
| 10237 |
SDVTList VTs = getVTList(VT); |
0 |
10237 |
SDVTList VTs = getVTList(VT); |
0 |
| 10238 |
SDValue Ops[] = { Op1, Op2 }; |
0 |
10238 |
SDValue Ops[] = { Op1, Op2 }; |
0 |
| 10239 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
10239 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
| 10240 |
} |
--- |
10240 |
} |
--- |
| 10241 |
|
--- |
10241 |
|
--- |
| 10242 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
10242 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
| 10243 |
EVT VT, SDValue Op1, SDValue Op2, |
--- |
10243 |
EVT VT, SDValue Op1, SDValue Op2, |
--- |
| 10244 |
SDValue Op3) { |
--- |
10244 |
SDValue Op3) { |
--- |
| 10245 |
SDVTList VTs = getVTList(VT); |
0 |
10245 |
SDVTList VTs = getVTList(VT); |
0 |
| 10246 |
SDValue Ops[] = { Op1, Op2, Op3 }; |
0 |
10246 |
SDValue Ops[] = { Op1, Op2, Op3 }; |
0 |
| 10247 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
10247 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
| 10248 |
} |
--- |
10248 |
} |
--- |
| 10249 |
|
--- |
10249 |
|
--- |
| 10250 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
10250 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
| 10251 |
EVT VT, ArrayRef Ops) { |
--- |
10251 |
EVT VT, ArrayRef Ops) { |
--- |
| 10252 |
SDVTList VTs = getVTList(VT); |
0 |
10252 |
SDVTList VTs = getVTList(VT); |
0 |
| 10253 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
10253 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
| 10254 |
} |
--- |
10254 |
} |
--- |
| 10255 |
|
--- |
10255 |
|
--- |
| 10256 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
10256 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
| 10257 |
EVT VT1, EVT VT2, SDValue Op1, |
--- |
10257 |
EVT VT1, EVT VT2, SDValue Op1, |
--- |
| 10258 |
SDValue Op2) { |
--- |
10258 |
SDValue Op2) { |
--- |
| 10259 |
SDVTList VTs = getVTList(VT1, VT2); |
0 |
10259 |
SDVTList VTs = getVTList(VT1, VT2); |
0 |
| 10260 |
SDValue Ops[] = { Op1, Op2 }; |
0 |
10260 |
SDValue Ops[] = { Op1, Op2 }; |
0 |
| 10261 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
10261 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
| 10262 |
} |
--- |
10262 |
} |
--- |
| 10263 |
|
--- |
10263 |
|
--- |
| 10264 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
10264 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
| 10265 |
EVT VT1, EVT VT2, SDValue Op1, |
--- |
10265 |
EVT VT1, EVT VT2, SDValue Op1, |
--- |
| 10266 |
SDValue Op2, SDValue Op3) { |
--- |
10266 |
SDValue Op2, SDValue Op3) { |
--- |
| 10267 |
SDVTList VTs = getVTList(VT1, VT2); |
0 |
10267 |
SDVTList VTs = getVTList(VT1, VT2); |
0 |
| 10268 |
SDValue Ops[] = { Op1, Op2, Op3 }; |
0 |
10268 |
SDValue Ops[] = { Op1, Op2, Op3 }; |
0 |
| 10269 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
10269 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
| 10270 |
} |
--- |
10270 |
} |
--- |
| 10271 |
|
--- |
10271 |
|
--- |
| 10272 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
10272 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
| 10273 |
EVT VT1, EVT VT2, |
--- |
10273 |
EVT VT1, EVT VT2, |
--- |
| 10274 |
ArrayRef Ops) { |
--- |
10274 |
ArrayRef Ops) { |
--- |
| 10275 |
SDVTList VTs = getVTList(VT1, VT2); |
0 |
10275 |
SDVTList VTs = getVTList(VT1, VT2); |
0 |
| 10276 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
10276 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
| 10277 |
} |
--- |
10277 |
} |
--- |
| 10278 |
|
--- |
10278 |
|
--- |
| 10279 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
10279 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
| 10280 |
EVT VT1, EVT VT2, EVT VT3, |
--- |
10280 |
EVT VT1, EVT VT2, EVT VT3, |
--- |
| 10281 |
SDValue Op1, SDValue Op2) { |
--- |
10281 |
SDValue Op1, SDValue Op2) { |
--- |
| 10282 |
SDVTList VTs = getVTList(VT1, VT2, VT3); |
0 |
10282 |
SDVTList VTs = getVTList(VT1, VT2, VT3); |
0 |
| 10283 |
SDValue Ops[] = { Op1, Op2 }; |
0 |
10283 |
SDValue Ops[] = { Op1, Op2 }; |
0 |
| 10284 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
10284 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
| 10285 |
} |
--- |
10285 |
} |
--- |
| 10286 |
|
--- |
10286 |
|
--- |
| 10287 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
10287 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
| 10288 |
EVT VT1, EVT VT2, EVT VT3, |
--- |
10288 |
EVT VT1, EVT VT2, EVT VT3, |
--- |
| 10289 |
SDValue Op1, SDValue Op2, |
--- |
10289 |
SDValue Op1, SDValue Op2, |
--- |
| 10290 |
SDValue Op3) { |
--- |
10290 |
SDValue Op3) { |
--- |
| 10291 |
SDVTList VTs = getVTList(VT1, VT2, VT3); |
0 |
10291 |
SDVTList VTs = getVTList(VT1, VT2, VT3); |
0 |
| 10292 |
SDValue Ops[] = { Op1, Op2, Op3 }; |
0 |
10292 |
SDValue Ops[] = { Op1, Op2, Op3 }; |
0 |
| 10293 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
10293 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
| 10294 |
} |
--- |
10294 |
} |
--- |
| 10295 |
|
--- |
10295 |
|
--- |
| 10296 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
10296 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
| 10297 |
EVT VT1, EVT VT2, EVT VT3, |
--- |
10297 |
EVT VT1, EVT VT2, EVT VT3, |
--- |
| 10298 |
ArrayRef Ops) { |
--- |
10298 |
ArrayRef Ops) { |
--- |
| 10299 |
SDVTList VTs = getVTList(VT1, VT2, VT3); |
0 |
10299 |
SDVTList VTs = getVTList(VT1, VT2, VT3); |
0 |
| 10300 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
10300 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
| 10301 |
} |
--- |
10301 |
} |
--- |
| 10302 |
|
--- |
10302 |
|
--- |
| 10303 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
10303 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
0 |
| 10304 |
ArrayRef ResultTys, |
--- |
10304 |
ArrayRef ResultTys, |
--- |
| 10305 |
ArrayRef Ops) { |
--- |
10305 |
ArrayRef Ops) { |
--- |
| 10306 |
SDVTList VTs = getVTList(ResultTys); |
0 |
10306 |
SDVTList VTs = getVTList(ResultTys); |
0 |
| 10307 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
10307 |
return getMachineNode(Opcode, dl, VTs, Ops); |
0 |
| 10308 |
} |
--- |
10308 |
} |
--- |
| 10309 |
|
--- |
10309 |
|
--- |
| 10310 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL, |
1 |
10310 |
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL, |
1 |
| 10311 |
SDVTList VTs, |
--- |
10311 |
SDVTList VTs, |
--- |
| 10312 |
ArrayRef Ops) { |
--- |
10312 |
ArrayRef Ops) { |
--- |
| 10313 |
bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; |
1 |
10313 |
bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; |
1 |
| 10314 |
MachineSDNode *N; |
--- |
10314 |
MachineSDNode *N; |
--- |
| 10315 |
void *IP = nullptr; |
1 |
10315 |
void *IP = nullptr; |
1 |
| 10316 |
|
--- |
10316 |
|
--- |
| 10317 |
if (DoCSE) { |
1 |
10317 |
if (DoCSE) { |
1 |
| 10318 |
FoldingSetNodeID ID; |
1 |
10318 |
FoldingSetNodeID ID; |
1 |
| 10319 |
AddNodeIDNode(ID, ~Opcode, VTs, Ops); |
1 |
10319 |
AddNodeIDNode(ID, ~Opcode, VTs, Ops); |
1 |
| 10320 |
IP = nullptr; |
1 |
10320 |
IP = nullptr; |
1 |
| 10321 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { |
1 |
10321 |
if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) { |
1 |
| 10322 |
return cast(UpdateSDLocOnMergeSDNode(E, DL)); |
0 |
10322 |
return cast(UpdateSDLocOnMergeSDNode(E, DL)); |
0 |
| 10323 |
} |
--- |
10323 |
} |
--- |
| 10324 |
} |
1 |
10324 |
} |
1 |
| 10325 |
|
--- |
10325 |
|
--- |
| 10326 |
// Allocate a new MachineSDNode. |
--- |
10326 |
// Allocate a new MachineSDNode. |
--- |
| 10327 |
N = newSDNode(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
1 |
10327 |
N = newSDNode(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
1 |
| 10328 |
createOperands(N, Ops); |
1 |
10328 |
createOperands(N, Ops); |
1 |
| 10329 |
|
--- |
10329 |
|
--- |
| 10330 |
if (DoCSE) |
1 |
10330 |
if (DoCSE) |
1 |
| 10331 |
CSEMap.InsertNode(N, IP); |
1 |
10331 |
CSEMap.InsertNode(N, IP); |
1 |
| 10332 |
|
--- |
10332 |
|
--- |
| 10333 |
InsertNode(N); |
1 |
10333 |
InsertNode(N); |
1 |
| 10334 |
NewSDValueDbgMsg(SDValue(N, 0), "Creating new machine node: ", this); |
1 |
10334 |
NewSDValueDbgMsg(SDValue(N, 0), "Creating new machine node: ", this); |
1 |
| 10335 |
return N; |
1 |
10335 |
return N; |
1 |
| 10336 |
} |
--- |
10336 |
} |
--- |
| 10337 |
|
--- |
10337 |
|
--- |
| 10338 |
/// getTargetExtractSubreg - A convenience function for creating |
--- |
10338 |
/// getTargetExtractSubreg - A convenience function for creating |
--- |
| 10339 |
/// TargetOpcode::EXTRACT_SUBREG nodes. |
--- |
10339 |
/// TargetOpcode::EXTRACT_SUBREG nodes. |
--- |
| 10340 |
SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, |
0 |
10340 |
SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, |
0 |
| 10341 |
SDValue Operand) { |
--- |
10341 |
SDValue Operand) { |
--- |
| 10342 |
SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32); |
0 |
10342 |
SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32); |
0 |
| 10343 |
SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, |
0 |
10343 |
SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, |
0 |
| 10344 |
VT, Operand, SRIdxVal); |
--- |
10344 |
VT, Operand, SRIdxVal); |
--- |
| 10345 |
return SDValue(Subreg, 0); |
0 |
10345 |
return SDValue(Subreg, 0); |
0 |
| 10346 |
} |
--- |
10346 |
} |
--- |
| 10347 |
|
--- |
10347 |
|
--- |
| 10348 |
/// getTargetInsertSubreg - A convenience function for creating |
--- |
10348 |
/// getTargetInsertSubreg - A convenience function for creating |
--- |
| 10349 |
/// TargetOpcode::INSERT_SUBREG nodes. |
--- |
10349 |
/// TargetOpcode::INSERT_SUBREG nodes. |
--- |
| 10350 |
SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, |
0 |
10350 |
SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, |
0 |
| 10351 |
SDValue Operand, SDValue Subreg) { |
--- |
10351 |
SDValue Operand, SDValue Subreg) { |
--- |
| 10352 |
SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32); |
0 |
10352 |
SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32); |
0 |
| 10353 |
SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL, |
0 |
10353 |
SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL, |
0 |
| 10354 |
VT, Operand, Subreg, SRIdxVal); |
--- |
10354 |
VT, Operand, Subreg, SRIdxVal); |
--- |
| 10355 |
return SDValue(Result, 0); |
0 |
10355 |
return SDValue(Result, 0); |
0 |
| 10356 |
} |
--- |
10356 |
} |
--- |
| 10357 |
|
--- |
10357 |
|
--- |
| 10358 |
/// getNodeIfExists - Get the specified node if it's already available, or |
--- |
10358 |
/// getNodeIfExists - Get the specified node if it's already available, or |
--- |
| 10359 |
/// else return NULL. |
--- |
10359 |
/// else return NULL. |
--- |
| 10360 |
SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, |
0 |
10360 |
SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, |
0 |
| 10361 |
ArrayRef Ops) { |
--- |
10361 |
ArrayRef Ops) { |
--- |
| 10362 |
SDNodeFlags Flags; |
0 |
10362 |
SDNodeFlags Flags; |
0 |
| 10363 |
if (Inserter) |
0 |
10363 |
if (Inserter) |
0 |
| 10364 |
Flags = Inserter->getFlags(); |
0 |
10364 |
Flags = Inserter->getFlags(); |
0 |
| 10365 |
return getNodeIfExists(Opcode, VTList, Ops, Flags); |
0 |
10365 |
return getNodeIfExists(Opcode, VTList, Ops, Flags); |
0 |
| 10366 |
} |
--- |
10366 |
} |
--- |
| 10367 |
|
--- |
10367 |
|
--- |
| 10368 |
SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, |
0 |
10368 |
SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, |
0 |
| 10369 |
ArrayRef Ops, |
--- |
10369 |
ArrayRef Ops, |
--- |
| 10370 |
const SDNodeFlags Flags) { |
--- |
10370 |
const SDNodeFlags Flags) { |
--- |
| 10371 |
if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { |
0 |
10371 |
if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { |
0 |
| 10372 |
FoldingSetNodeID ID; |
0 |
10372 |
FoldingSetNodeID ID; |
0 |
| 10373 |
AddNodeIDNode(ID, Opcode, VTList, Ops); |
0 |
10373 |
AddNodeIDNode(ID, Opcode, VTList, Ops); |
0 |
| 10374 |
void *IP = nullptr; |
0 |
10374 |
void *IP = nullptr; |
0 |
| 10375 |
if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) { |
0 |
10375 |
if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) { |
0 |
| 10376 |
E->intersectFlagsWith(Flags); |
0 |
10376 |
E->intersectFlagsWith(Flags); |
0 |
| 10377 |
return E; |
0 |
10377 |
return E; |
0 |
| 10378 |
} |
--- |
10378 |
} |
--- |
| 10379 |
} |
0 |
10379 |
} |
0 |
| 10380 |
return nullptr; |
0 |
10380 |
return nullptr; |
0 |
| 10381 |
} |
--- |
10381 |
} |
--- |
| 10382 |
|
--- |
10382 |
|
--- |
| 10383 |
/// doesNodeExist - Check if a node exists without modifying its flags. |
--- |
10383 |
/// doesNodeExist - Check if a node exists without modifying its flags. |
--- |
| 10384 |
bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList, |
3 |
10384 |
bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList, |
3 |
| 10385 |
ArrayRef Ops) { |
--- |
10385 |
ArrayRef Ops) { |
--- |
| 10386 |
if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { |
3 |
10386 |
if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { |
3 |
| 10387 |
FoldingSetNodeID ID; |
3 |
10387 |
FoldingSetNodeID ID; |
3 |
| 10388 |
AddNodeIDNode(ID, Opcode, VTList, Ops); |
3 |
10388 |
AddNodeIDNode(ID, Opcode, VTList, Ops); |
3 |
| 10389 |
void *IP = nullptr; |
3 |
10389 |
void *IP = nullptr; |
3 |
| 10390 |
if (FindNodeOrInsertPos(ID, SDLoc(), IP)) |
3 |
10390 |
if (FindNodeOrInsertPos(ID, SDLoc(), IP)) |
3 |
| 10391 |
return true; |
0 |
10391 |
return true; |
0 |
| 10392 |
} |
3 |
10392 |
} |
3 |
| 10393 |
return false; |
3 |
10393 |
return false; |
3 |
| 10394 |
} |
--- |
10394 |
} |
--- |
| 10395 |
|
--- |
10395 |
|
--- |
| 10396 |
/// getDbgValue - Creates a SDDbgValue node. |
--- |
10396 |
/// getDbgValue - Creates a SDDbgValue node. |
--- |
| 10397 |
/// |
--- |
10397 |
/// |
--- |
| 10398 |
/// SDNode |
--- |
10398 |
/// SDNode |
--- |
| 10399 |
SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr, |
0 |
10399 |
SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr, |
0 |
| 10400 |
SDNode *N, unsigned R, bool IsIndirect, |
--- |
10400 |
SDNode *N, unsigned R, bool IsIndirect, |
--- |
| 10401 |
const DebugLoc &DL, unsigned O) { |
--- |
10401 |
const DebugLoc &DL, unsigned O) { |
--- |
| 10402 |
assert(cast(Var)->isValidLocationForIntrinsic(DL) && |
0 |
10402 |
assert(cast(Var)->isValidLocationForIntrinsic(DL) && |
0 |
| 10403 |
"Expected inlined-at fields to agree"); |
--- |
10403 |
"Expected inlined-at fields to agree"); |
--- |
| 10404 |
return new (DbgInfo->getAlloc()) |
0 |
10404 |
return new (DbgInfo->getAlloc()) |
0 |
| 10405 |
SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(N, R), |
0 |
10405 |
SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(N, R), |
0 |
| 10406 |
{}, IsIndirect, DL, O, |
--- |
10406 |
{}, IsIndirect, DL, O, |
--- |
| 10407 |
/*IsVariadic=*/false); |
0 |
10407 |
/*IsVariadic=*/false); |
0 |
| 10408 |
} |
--- |
10408 |
} |
--- |
| 10409 |
|
--- |
10409 |
|
--- |
| 10410 |
/// Constant |
--- |
10410 |
/// Constant |
--- |
| 10411 |
SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var, |
0 |
10411 |
SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var, |
0 |
| 10412 |
DIExpression *Expr, |
--- |
10412 |
DIExpression *Expr, |
--- |
| 10413 |
const Value *C, |
--- |
10413 |
const Value *C, |
--- |
| 10414 |
const DebugLoc &DL, unsigned O) { |
--- |
10414 |
const DebugLoc &DL, unsigned O) { |
--- |
| 10415 |
assert(cast(Var)->isValidLocationForIntrinsic(DL) && |
0 |
10415 |
assert(cast(Var)->isValidLocationForIntrinsic(DL) && |
0 |
| 10416 |
"Expected inlined-at fields to agree"); |
--- |
10416 |
"Expected inlined-at fields to agree"); |
--- |
| 10417 |
return new (DbgInfo->getAlloc()) |
0 |
10417 |
return new (DbgInfo->getAlloc()) |
0 |
| 10418 |
SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(C), {}, |
0 |
10418 |
SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(C), {}, |
0 |
| 10419 |
/*IsIndirect=*/false, DL, O, |
--- |
10419 |
/*IsIndirect=*/false, DL, O, |
--- |
| 10420 |
/*IsVariadic=*/false); |
0 |
10420 |
/*IsVariadic=*/false); |
0 |
| 10421 |
} |
--- |
10421 |
} |
--- |
| 10422 |
|
--- |
10422 |
|
--- |
| 10423 |
/// FrameIndex |
--- |
10423 |
/// FrameIndex |
--- |
| 10424 |
SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var, |
0 |
10424 |
SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var, |
0 |
| 10425 |
DIExpression *Expr, unsigned FI, |
--- |
10425 |
DIExpression *Expr, unsigned FI, |
--- |
| 10426 |
bool IsIndirect, |
--- |
10426 |
bool IsIndirect, |
--- |
| 10427 |
const DebugLoc &DL, |
--- |
10427 |
const DebugLoc &DL, |
--- |
| 10428 |
unsigned O) { |
--- |
10428 |
unsigned O) { |
--- |
| 10429 |
assert(cast(Var)->isValidLocationForIntrinsic(DL) && |
0 |
10429 |
assert(cast(Var)->isValidLocationForIntrinsic(DL) && |
0 |
| 10430 |
"Expected inlined-at fields to agree"); |
--- |
10430 |
"Expected inlined-at fields to agree"); |
--- |
| 10431 |
return getFrameIndexDbgValue(Var, Expr, FI, {}, IsIndirect, DL, O); |
0 |
10431 |
return getFrameIndexDbgValue(Var, Expr, FI, {}, IsIndirect, DL, O); |
0 |
| 10432 |
} |
--- |
10432 |
} |
--- |
| 10433 |
|
--- |
10433 |
|
--- |
| 10434 |
/// FrameIndex with dependencies |
--- |
10434 |
/// FrameIndex with dependencies |
--- |
| 10435 |
SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var, |
0 |
10435 |
SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var, |
0 |
| 10436 |
DIExpression *Expr, unsigned FI, |
--- |
10436 |
DIExpression *Expr, unsigned FI, |
--- |
| 10437 |
ArrayRef Dependencies, |
--- |
10437 |
ArrayRef Dependencies, |
--- |
| 10438 |
bool IsIndirect, |
--- |
10438 |
bool IsIndirect, |
--- |
| 10439 |
const DebugLoc &DL, |
--- |
10439 |
const DebugLoc &DL, |
--- |
| 10440 |
unsigned O) { |
--- |
10440 |
unsigned O) { |
--- |
| 10441 |
assert(cast(Var)->isValidLocationForIntrinsic(DL) && |
0 |
10441 |
assert(cast(Var)->isValidLocationForIntrinsic(DL) && |
0 |
| 10442 |
"Expected inlined-at fields to agree"); |
--- |
10442 |
"Expected inlined-at fields to agree"); |
--- |
| 10443 |
return new (DbgInfo->getAlloc()) |
0 |
10443 |
return new (DbgInfo->getAlloc()) |
0 |
| 10444 |
SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FI), |
0 |
10444 |
SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FI), |
0 |
| 10445 |
Dependencies, IsIndirect, DL, O, |
--- |
10445 |
Dependencies, IsIndirect, DL, O, |
--- |
| 10446 |
/*IsVariadic=*/false); |
0 |
10446 |
/*IsVariadic=*/false); |
0 |
| 10447 |
} |
--- |
10447 |
} |
--- |
| 10448 |
|
--- |
10448 |
|
--- |
| 10449 |
/// VReg |
--- |
10449 |
/// VReg |
--- |
| 10450 |
SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr, |
0 |
10450 |
SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr, |
0 |
| 10451 |
unsigned VReg, bool IsIndirect, |
--- |
10451 |
unsigned VReg, bool IsIndirect, |
--- |
| 10452 |
const DebugLoc &DL, unsigned O) { |
--- |
10452 |
const DebugLoc &DL, unsigned O) { |
--- |
| 10453 |
assert(cast(Var)->isValidLocationForIntrinsic(DL) && |
0 |
10453 |
assert(cast(Var)->isValidLocationForIntrinsic(DL) && |
0 |
| 10454 |
"Expected inlined-at fields to agree"); |
--- |
10454 |
"Expected inlined-at fields to agree"); |
--- |
| 10455 |
return new (DbgInfo->getAlloc()) |
0 |
10455 |
return new (DbgInfo->getAlloc()) |
0 |
| 10456 |
SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg), |
0 |
10456 |
SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg), |
0 |
| 10457 |
{}, IsIndirect, DL, O, |
--- |
10457 |
{}, IsIndirect, DL, O, |
--- |
| 10458 |
/*IsVariadic=*/false); |
0 |
10458 |
/*IsVariadic=*/false); |
0 |
| 10459 |
} |
--- |
10459 |
} |
--- |
| 10460 |
|
--- |
10460 |
|
--- |
| 10461 |
SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr, |
0 |
10461 |
SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr, |
0 |
| 10462 |
ArrayRef Locs, |
--- |
10462 |
ArrayRef Locs, |
--- |
| 10463 |
ArrayRef Dependencies, |
--- |
10463 |
ArrayRef Dependencies, |
--- |
| 10464 |
bool IsIndirect, const DebugLoc &DL, |
--- |
10464 |
bool IsIndirect, const DebugLoc &DL, |
--- |
| 10465 |
unsigned O, bool IsVariadic) { |
--- |
10465 |
unsigned O, bool IsVariadic) { |
--- |
| 10466 |
assert(cast(Var)->isValidLocationForIntrinsic(DL) && |
0 |
10466 |
assert(cast(Var)->isValidLocationForIntrinsic(DL) && |
0 |
| 10467 |
"Expected inlined-at fields to agree"); |
--- |
10467 |
"Expected inlined-at fields to agree"); |
--- |
| 10468 |
return new (DbgInfo->getAlloc()) |
0 |
10468 |
return new (DbgInfo->getAlloc()) |
0 |
| 10469 |
SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect, |
0 |
10469 |
SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect, |
0 |
| 10470 |
DL, O, IsVariadic); |
0 |
10470 |
DL, O, IsVariadic); |
0 |
| 10471 |
} |
--- |
10471 |
} |
--- |
| 10472 |
|
--- |
10472 |
|
--- |
| 10473 |
void SelectionDAG::transferDbgValues(SDValue From, SDValue To, |
4 |
10473 |
void SelectionDAG::transferDbgValues(SDValue From, SDValue To, |
4 |
| 10474 |
unsigned OffsetInBits, unsigned SizeInBits, |
--- |
10474 |
unsigned OffsetInBits, unsigned SizeInBits, |
--- |
| 10475 |
bool InvalidateDbg) { |
--- |
10475 |
bool InvalidateDbg) { |
--- |
| 10476 |
SDNode *FromNode = From.getNode(); |
4 |
10476 |
SDNode *FromNode = From.getNode(); |
4 |
| 10477 |
SDNode *ToNode = To.getNode(); |
4 |
10477 |
SDNode *ToNode = To.getNode(); |
4 |
| 10478 |
assert(FromNode && ToNode && "Can't modify dbg values"); |
4 |
10478 |
assert(FromNode && ToNode && "Can't modify dbg values"); |
4 |
| 10479 |
|
--- |
10479 |
|
--- |
| 10480 |
// PR35338 |
--- |
10480 |
// PR35338 |
--- |
| 10481 |
// TODO: assert(From != To && "Redundant dbg value transfer"); |
--- |
10481 |
// TODO: assert(From != To && "Redundant dbg value transfer"); |
--- |
| 10482 |
// TODO: assert(FromNode != ToNode && "Intranode dbg value transfer"); |
--- |
10482 |
// TODO: assert(FromNode != ToNode && "Intranode dbg value transfer"); |
--- |
| 10483 |
if (From == To || FromNode == ToNode) |
4 |
10483 |
if (From == To || FromNode == ToNode) |
4 |
| 10484 |
return; |
4 |
10484 |
return; |
4 |
| 10485 |
|
--- |
10485 |
|
--- |
| 10486 |
if (!FromNode->getHasDebugValue()) |
4 |
10486 |
if (!FromNode->getHasDebugValue()) |
4 |
| 10487 |
return; |
4 |
10487 |
return; |
4 |
| 10488 |
|
--- |
10488 |
|
--- |
| 10489 |
SDDbgOperand FromLocOp = |
--- |
10489 |
SDDbgOperand FromLocOp = |
--- |
| 10490 |
SDDbgOperand::fromNode(From.getNode(), From.getResNo()); |
0 |
10490 |
SDDbgOperand::fromNode(From.getNode(), From.getResNo()); |
0 |
| 10491 |
SDDbgOperand ToLocOp = SDDbgOperand::fromNode(To.getNode(), To.getResNo()); |
0 |
10491 |
SDDbgOperand ToLocOp = SDDbgOperand::fromNode(To.getNode(), To.getResNo()); |
0 |
| 10492 |
|
--- |
10492 |
|
--- |
| 10493 |
SmallVector ClonedDVs; |
0 |
10493 |
SmallVector ClonedDVs; |
0 |
| 10494 |
for (SDDbgValue *Dbg : GetDbgValues(FromNode)) { |
0 |
10494 |
for (SDDbgValue *Dbg : GetDbgValues(FromNode)) { |
0 |
| 10495 |
if (Dbg->isInvalidated()) |
0 |
10495 |
if (Dbg->isInvalidated()) |
0 |
| 10496 |
continue; |
0 |
10496 |
continue; |
0 |
| 10497 |
|
--- |
10497 |
|
--- |
| 10498 |
// TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value"); |
--- |
10498 |
// TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value"); |
--- |
| 10499 |
|
--- |
10499 |
|
--- |
| 10500 |
// Create a new location ops vector that is equal to the old vector, but |
--- |
10500 |
// Create a new location ops vector that is equal to the old vector, but |
--- |
| 10501 |
// with each instance of FromLocOp replaced with ToLocOp. |
--- |
10501 |
// with each instance of FromLocOp replaced with ToLocOp. |
--- |
| 10502 |
bool Changed = false; |
0 |
10502 |
bool Changed = false; |
0 |
| 10503 |
auto NewLocOps = Dbg->copyLocationOps(); |
0 |
10503 |
auto NewLocOps = Dbg->copyLocationOps(); |
0 |
| 10504 |
std::replace_if( |
0 |
10504 |
std::replace_if( |
0 |
| 10505 |
NewLocOps.begin(), NewLocOps.end(), |
--- |
10505 |
NewLocOps.begin(), NewLocOps.end(), |
--- |
| 10506 |
[&Changed, FromLocOp](const SDDbgOperand &Op) { |
0 |
10506 |
[&Changed, FromLocOp](const SDDbgOperand &Op) { |
0 |
| 10507 |
bool Match = Op == FromLocOp; |
0 |
10507 |
bool Match = Op == FromLocOp; |
0 |
| 10508 |
Changed |= Match; |
0 |
10508 |
Changed |= Match; |
0 |
| 10509 |
return Match; |
0 |
10509 |
return Match; |
0 |
| 10510 |
}, |
--- |
10510 |
}, |
--- |
| 10511 |
ToLocOp); |
--- |
10511 |
ToLocOp); |
--- |
| 10512 |
// Ignore this SDDbgValue if we didn't find a matching location. |
--- |
10512 |
// Ignore this SDDbgValue if we didn't find a matching location. |
--- |
| 10513 |
if (!Changed) |
0 |
10513 |
if (!Changed) |
0 |
| 10514 |
continue; |
0 |
10514 |
continue; |
0 |
| 10515 |
|
--- |
10515 |
|
--- |
| 10516 |
DIVariable *Var = Dbg->getVariable(); |
0 |
10516 |
DIVariable *Var = Dbg->getVariable(); |
0 |
| 10517 |
auto *Expr = Dbg->getExpression(); |
0 |
10517 |
auto *Expr = Dbg->getExpression(); |
0 |
| 10518 |
// If a fragment is requested, update the expression. |
--- |
10518 |
// If a fragment is requested, update the expression. |
--- |
| 10519 |
if (SizeInBits) { |
0 |
10519 |
if (SizeInBits) { |
0 |
| 10520 |
// When splitting a larger (e.g., sign-extended) value whose |
--- |
10520 |
// When splitting a larger (e.g., sign-extended) value whose |
--- |
| 10521 |
// lower bits are described with an SDDbgValue, do not attempt |
--- |
10521 |
// lower bits are described with an SDDbgValue, do not attempt |
--- |
| 10522 |
// to transfer the SDDbgValue to the upper bits. |
--- |
10522 |
// to transfer the SDDbgValue to the upper bits. |
--- |
| 10523 |
if (auto FI = Expr->getFragmentInfo()) |
0 |
10523 |
if (auto FI = Expr->getFragmentInfo()) |
0 |
| 10524 |
if (OffsetInBits + SizeInBits > FI->SizeInBits) |
0 |
10524 |
if (OffsetInBits + SizeInBits > FI->SizeInBits) |
0 |
| 10525 |
continue; |
0 |
10525 |
continue; |
0 |
| 10526 |
auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits, |
0 |
10526 |
auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits, |
0 |
| 10527 |
SizeInBits); |
--- |
10527 |
SizeInBits); |
--- |
| 10528 |
if (!Fragment) |
0 |
10528 |
if (!Fragment) |
0 |
| 10529 |
continue; |
0 |
10529 |
continue; |
0 |
| 10530 |
Expr = *Fragment; |
0 |
10530 |
Expr = *Fragment; |
0 |
| 10531 |
} |
--- |
10531 |
} |
--- |
| 10532 |
|
--- |
10532 |
|
--- |
| 10533 |
auto AdditionalDependencies = Dbg->getAdditionalDependencies(); |
0 |
10533 |
auto AdditionalDependencies = Dbg->getAdditionalDependencies(); |
0 |
| 10534 |
// Clone the SDDbgValue and move it to To. |
--- |
10534 |
// Clone the SDDbgValue and move it to To. |
--- |
| 10535 |
SDDbgValue *Clone = getDbgValueList( |
0 |
10535 |
SDDbgValue *Clone = getDbgValueList( |
0 |
| 10536 |
Var, Expr, NewLocOps, AdditionalDependencies, Dbg->isIndirect(), |
0 |
10536 |
Var, Expr, NewLocOps, AdditionalDependencies, Dbg->isIndirect(), |
0 |
| 10537 |
Dbg->getDebugLoc(), std::max(ToNode->getIROrder(), Dbg->getOrder()), |
0 |
10537 |
Dbg->getDebugLoc(), std::max(ToNode->getIROrder(), Dbg->getOrder()), |
0 |
| 10538 |
Dbg->isVariadic()); |
0 |
10538 |
Dbg->isVariadic()); |
0 |
| 10539 |
ClonedDVs.push_back(Clone); |
0 |
10539 |
ClonedDVs.push_back(Clone); |
0 |
| 10540 |
|
--- |
10540 |
|
--- |
| 10541 |
if (InvalidateDbg) { |
0 |
10541 |
if (InvalidateDbg) { |
0 |
| 10542 |
// Invalidate value and indicate the SDDbgValue should not be emitted. |
--- |
10542 |
// Invalidate value and indicate the SDDbgValue should not be emitted. |
--- |
| 10543 |
Dbg->setIsInvalidated(); |
0 |
10543 |
Dbg->setIsInvalidated(); |
0 |
| 10544 |
Dbg->setIsEmitted(); |
0 |
10544 |
Dbg->setIsEmitted(); |
0 |
| 10545 |
} |
--- |
10545 |
} |
--- |
| 10546 |
} |
0 |
10546 |
} |
0 |
| 10547 |
|
--- |
10547 |
|
--- |
| 10548 |
for (SDDbgValue *Dbg : ClonedDVs) { |
0 |
10548 |
for (SDDbgValue *Dbg : ClonedDVs) { |
0 |
| 10549 |
assert(is_contained(Dbg->getSDNodes(), ToNode) && |
0 |
10549 |
assert(is_contained(Dbg->getSDNodes(), ToNode) && |
0 |
| 10550 |
"Transferred DbgValues should depend on the new SDNode"); |
--- |
10550 |
"Transferred DbgValues should depend on the new SDNode"); |
--- |
| 10551 |
AddDbgValue(Dbg, false); |
0 |
10551 |
AddDbgValue(Dbg, false); |
0 |
| 10552 |
} |
--- |
10552 |
} |
--- |
| 10553 |
} |
0 |
10553 |
} |
0 |
| 10554 |
|
--- |
10554 |
|
--- |
| 10555 |
void SelectionDAG::salvageDebugInfo(SDNode &N) { |
13 |
10555 |
void SelectionDAG::salvageDebugInfo(SDNode &N) { |
13 |
| 10556 |
if (!N.getHasDebugValue()) |
13 |
10556 |
if (!N.getHasDebugValue()) |
13 |
| 10557 |
return; |
13 |
10557 |
return; |
13 |
| 10558 |
|
--- |
10558 |
|
--- |
| 10559 |
SmallVector ClonedDVs; |
0 |
10559 |
SmallVector ClonedDVs; |
0 |
| 10560 |
for (auto *DV : GetDbgValues(&N)) { |
0 |
10560 |
for (auto *DV : GetDbgValues(&N)) { |
0 |
| 10561 |
if (DV->isInvalidated()) |
0 |
10561 |
if (DV->isInvalidated()) |
0 |
| 10562 |
continue; |
0 |
10562 |
continue; |
0 |
| 10563 |
switch (N.getOpcode()) { |
0 |
10563 |
switch (N.getOpcode()) { |
0 |
| 10564 |
default: |
0 |
10564 |
default: |
0 |
| 10565 |
break; |
0 |
10565 |
break; |
0 |
| 10566 |
case ISD::ADD: |
0 |
10566 |
case ISD::ADD: |
0 |
| 10567 |
SDValue N0 = N.getOperand(0); |
0 |
10567 |
SDValue N0 = N.getOperand(0); |
0 |
| 10568 |
SDValue N1 = N.getOperand(1); |
0 |
10568 |
SDValue N1 = N.getOperand(1); |
0 |
| 10569 |
if (!isa(N0) && isa(N1)) { |
0 |
10569 |
if (!isa(N0) && isa(N1)) { |
0 |
| 10570 |
uint64_t Offset = N.getConstantOperandVal(1); |
0 |
10570 |
uint64_t Offset = N.getConstantOperandVal(1); |
0 |
| 10571 |
|
--- |
10571 |
|
--- |
| 10572 |
// Rewrite an ADD constant node into a DIExpression. Since we are |
--- |
10572 |
// Rewrite an ADD constant node into a DIExpression. Since we are |
--- |
| 10573 |
// performing arithmetic to compute the variable's *value* in the |
--- |
10573 |
// performing arithmetic to compute the variable's *value* in the |
--- |
| 10574 |
// DIExpression, we need to mark the expression with a |
--- |
10574 |
// DIExpression, we need to mark the expression with a |
--- |
| 10575 |
// DW_OP_stack_value. |
--- |
10575 |
// DW_OP_stack_value. |
--- |
| 10576 |
auto *DIExpr = DV->getExpression(); |
0 |
10576 |
auto *DIExpr = DV->getExpression(); |
0 |
| 10577 |
auto NewLocOps = DV->copyLocationOps(); |
0 |
10577 |
auto NewLocOps = DV->copyLocationOps(); |
0 |
| 10578 |
bool Changed = false; |
0 |
10578 |
bool Changed = false; |
0 |
| 10579 |
for (size_t i = 0; i < NewLocOps.size(); ++i) { |
0 |
10579 |
for (size_t i = 0; i < NewLocOps.size(); ++i) { |
0 |
| 10580 |
// We're not given a ResNo to compare against because the whole |
--- |
10580 |
// We're not given a ResNo to compare against because the whole |
--- |
| 10581 |
// node is going away. We know that any ISD::ADD only has one |
--- |
10581 |
// node is going away. We know that any ISD::ADD only has one |
--- |
| 10582 |
// result, so we can assume any node match is using the result. |
--- |
10582 |
// result, so we can assume any node match is using the result. |
--- |
| 10583 |
if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE || |
0 |
10583 |
if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE || |
0 |
| 10584 |
NewLocOps[i].getSDNode() != &N) |
0 |
10584 |
NewLocOps[i].getSDNode() != &N) |
0 |
| 10585 |
continue; |
0 |
10585 |
continue; |
0 |
| 10586 |
NewLocOps[i] = SDDbgOperand::fromNode(N0.getNode(), N0.getResNo()); |
0 |
10586 |
NewLocOps[i] = SDDbgOperand::fromNode(N0.getNode(), N0.getResNo()); |
0 |
| 10587 |
SmallVector ExprOps; |
0 |
10587 |
SmallVector ExprOps; |
0 |
| 10588 |
DIExpression::appendOffset(ExprOps, Offset); |
0 |
10588 |
DIExpression::appendOffset(ExprOps, Offset); |
0 |
| 10589 |
DIExpr = DIExpression::appendOpsToArg(DIExpr, ExprOps, i, true); |
0 |
10589 |
DIExpr = DIExpression::appendOpsToArg(DIExpr, ExprOps, i, true); |
0 |
| 10590 |
Changed = true; |
0 |
10590 |
Changed = true; |
0 |
| 10591 |
} |
0 |
10591 |
} |
0 |
| 10592 |
(void)Changed; |
--- |
10592 |
(void)Changed; |
--- |
| 10593 |
assert(Changed && "Salvage target doesn't use N"); |
0 |
10593 |
assert(Changed && "Salvage target doesn't use N"); |
0 |
| 10594 |
|
--- |
10594 |
|
--- |
| 10595 |
auto AdditionalDependencies = DV->getAdditionalDependencies(); |
0 |
10595 |
auto AdditionalDependencies = DV->getAdditionalDependencies(); |
0 |
| 10596 |
SDDbgValue *Clone = getDbgValueList(DV->getVariable(), DIExpr, |
0 |
10596 |
SDDbgValue *Clone = getDbgValueList(DV->getVariable(), DIExpr, |
0 |
| 10597 |
NewLocOps, AdditionalDependencies, |
--- |
10597 |
NewLocOps, AdditionalDependencies, |
--- |
| 10598 |
DV->isIndirect(), DV->getDebugLoc(), |
0 |
10598 |
DV->isIndirect(), DV->getDebugLoc(), |
0 |
| 10599 |
DV->getOrder(), DV->isVariadic()); |
0 |
10599 |
DV->getOrder(), DV->isVariadic()); |
0 |
| 10600 |
ClonedDVs.push_back(Clone); |
0 |
10600 |
ClonedDVs.push_back(Clone); |
0 |
| 10601 |
DV->setIsInvalidated(); |
0 |
10601 |
DV->setIsInvalidated(); |
0 |
| 10602 |
DV->setIsEmitted(); |
0 |
10602 |
DV->setIsEmitted(); |
0 |
| 10603 |
LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; |
0 |
10603 |
LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; |
0 |
| 10604 |
N0.getNode()->dumprFull(this); |
--- |
10604 |
N0.getNode()->dumprFull(this); |
--- |
| 10605 |
dbgs() << " into " << *DIExpr << '\n'); |
--- |
10605 |
dbgs() << " into " << *DIExpr << '\n'); |
--- |
| 10606 |
} |
0 |
10606 |
} |
0 |
| 10607 |
} |
--- |
10607 |
} |
--- |
| 10608 |
} |
--- |
10608 |
} |
--- |
| 10609 |
|
--- |
10609 |
|
--- |
| 10610 |
for (SDDbgValue *Dbg : ClonedDVs) { |
0 |
10610 |
for (SDDbgValue *Dbg : ClonedDVs) { |
0 |
| 10611 |
assert(!Dbg->getSDNodes().empty() && |
0 |
10611 |
assert(!Dbg->getSDNodes().empty() && |
0 |
| 10612 |
"Salvaged DbgValue should depend on a new SDNode"); |
--- |
10612 |
"Salvaged DbgValue should depend on a new SDNode"); |
--- |
| 10613 |
AddDbgValue(Dbg, false); |
0 |
10613 |
AddDbgValue(Dbg, false); |
0 |
| 10614 |
} |
--- |
10614 |
} |
--- |
| 10615 |
} |
0 |
10615 |
} |
0 |
| 10616 |
|
--- |
10616 |
|
--- |
| 10617 |
/// Creates a SDDbgLabel node. |
--- |
10617 |
/// Creates a SDDbgLabel node. |
--- |
| 10618 |
SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label, |
0 |
10618 |
SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label, |
0 |
| 10619 |
const DebugLoc &DL, unsigned O) { |
--- |
10619 |
const DebugLoc &DL, unsigned O) { |
--- |
| 10620 |
assert(cast(Label)->isValidLocationForIntrinsic(DL) && |
0 |
10620 |
assert(cast(Label)->isValidLocationForIntrinsic(DL) && |
0 |
| 10621 |
"Expected inlined-at fields to agree"); |
--- |
10621 |
"Expected inlined-at fields to agree"); |
--- |
| 10622 |
return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O); |
0 |
10622 |
return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O); |
0 |
| 10623 |
} |
--- |
10623 |
} |
--- |
| 10624 |
|
--- |
10624 |
|
--- |
| 10625 |
namespace { |
--- |
10625 |
namespace { |
--- |
| 10626 |
|
--- |
10626 |
|
--- |
| 10627 |
/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node |
--- |
10627 |
/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node |
--- |
| 10628 |
/// pointed to by a use iterator is deleted, increment the use iterator |
--- |
10628 |
/// pointed to by a use iterator is deleted, increment the use iterator |
--- |
| 10629 |
/// so that it doesn't dangle. |
--- |
10629 |
/// so that it doesn't dangle. |
--- |
| 10630 |
/// |
--- |
10630 |
/// |
--- |
| 10631 |
class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { |
--- |
10631 |
class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { |
--- |
| 10632 |
SDNode::use_iterator &UI; |
--- |
10632 |
SDNode::use_iterator &UI; |
--- |
| 10633 |
SDNode::use_iterator &UE; |
--- |
10633 |
SDNode::use_iterator &UE; |
--- |
| 10634 |
|
--- |
10634 |
|
--- |
| 10635 |
void NodeDeleted(SDNode *N, SDNode *E) override { |
0 |
10635 |
void NodeDeleted(SDNode *N, SDNode *E) override { |
0 |
| 10636 |
// Increment the iterator as needed. |
--- |
10636 |
// Increment the iterator as needed. |
--- |
| 10637 |
while (UI != UE && N == *UI) |
0 |
10637 |
while (UI != UE && N == *UI) |
0 |
| 10638 |
++UI; |
0 |
10638 |
++UI; |
0 |
| 10639 |
} |
0 |
10639 |
} |
0 |
| 10640 |
|
--- |
10640 |
|
--- |
| 10641 |
public: |
--- |
10641 |
public: |
--- |
| 10642 |
RAUWUpdateListener(SelectionDAG &d, |
3 |
10642 |
RAUWUpdateListener(SelectionDAG &d, |
3 |
| 10643 |
SDNode::use_iterator &ui, |
--- |
10643 |
SDNode::use_iterator &ui, |
--- |
| 10644 |
SDNode::use_iterator &ue) |
--- |
10644 |
SDNode::use_iterator &ue) |
--- |
| 10645 |
: SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {} |
3 |
10645 |
: SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {} |
3 |
| 10646 |
}; |
--- |
10646 |
}; |
--- |
| 10647 |
|
--- |
10647 |
|
--- |
| 10648 |
} // end anonymous namespace |
--- |
10648 |
} // end anonymous namespace |
--- |
| 10649 |
|
--- |
10649 |
|
--- |
| 10650 |
/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. |
--- |
10650 |
/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. |
--- |
| 10651 |
/// This can cause recursive merging of nodes in the DAG. |
--- |
10651 |
/// This can cause recursive merging of nodes in the DAG. |
--- |
| 10652 |
/// |
--- |
10652 |
/// |
--- |
| 10653 |
/// This version assumes From has a single result value. |
--- |
10653 |
/// This version assumes From has a single result value. |
--- |
| 10654 |
/// |
--- |
10654 |
/// |
--- |
| 10655 |
void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) { |
2 |
10655 |
void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) { |
2 |
| 10656 |
SDNode *From = FromN.getNode(); |
2 |
10656 |
SDNode *From = FromN.getNode(); |
2 |
| 10657 |
assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && |
2 |
10657 |
assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && |
2 |
| 10658 |
"Cannot replace with this method!"); |
--- |
10658 |
"Cannot replace with this method!"); |
--- |
| 10659 |
assert(From != To.getNode() && "Cannot replace uses of with self"); |
2 |
10659 |
assert(From != To.getNode() && "Cannot replace uses of with self"); |
2 |
| 10660 |
|
--- |
10660 |
|
--- |
| 10661 |
// Preserve Debug Values |
--- |
10661 |
// Preserve Debug Values |
--- |
| 10662 |
transferDbgValues(FromN, To); |
2 |
10662 |
transferDbgValues(FromN, To); |
2 |
| 10663 |
// Preserve extra info. |
--- |
10663 |
// Preserve extra info. |
--- |
| 10664 |
copyExtraInfo(From, To.getNode()); |
2 |
10664 |
copyExtraInfo(From, To.getNode()); |
2 |
| 10665 |
|
--- |
10665 |
|
--- |
| 10666 |
// Iterate over all the existing uses of From. New uses will be added |
--- |
10666 |
// Iterate over all the existing uses of From. New uses will be added |
--- |
| 10667 |
// to the beginning of the use list, which we avoid visiting. |
--- |
10667 |
// to the beginning of the use list, which we avoid visiting. |
--- |
| 10668 |
// This specifically avoids visiting uses of From that arise while the |
--- |
10668 |
// This specifically avoids visiting uses of From that arise while the |
--- |
| 10669 |
// replacement is happening, because any such uses would be the result |
--- |
10669 |
// replacement is happening, because any such uses would be the result |
--- |
| 10670 |
// of CSE: If an existing node looks like From after one of its operands |
--- |
10670 |
// of CSE: If an existing node looks like From after one of its operands |
--- |
| 10671 |
// is replaced by To, we don't want to replace of all its users with To |
--- |
10671 |
// is replaced by To, we don't want to replace of all its users with To |
--- |
| 10672 |
// too. See PR3018 for more info. |
--- |
10672 |
// too. See PR3018 for more info. |
--- |
| 10673 |
SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); |
2 |
10673 |
SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); |
2 |
| 10674 |
RAUWUpdateListener Listener(*this, UI, UE); |
2 |
10674 |
RAUWUpdateListener Listener(*this, UI, UE); |
2 |
| 10675 |
while (UI != UE) { |
4 |
10675 |
while (UI != UE) { |
4 |
| 10676 |
SDNode *User = *UI; |
2 |
10676 |
SDNode *User = *UI; |
2 |
| 10677 |
|
--- |
10677 |
|
--- |
| 10678 |
// This node is about to morph, remove its old self from the CSE maps. |
--- |
10678 |
// This node is about to morph, remove its old self from the CSE maps. |
--- |
| 10679 |
RemoveNodeFromCSEMaps(User); |
2 |
10679 |
RemoveNodeFromCSEMaps(User); |
2 |
| 10680 |
|
--- |
10680 |
|
--- |
| 10681 |
// A user can appear in a use list multiple times, and when this |
--- |
10681 |
// A user can appear in a use list multiple times, and when this |
--- |
| 10682 |
// happens the uses are usually next to each other in the list. |
--- |
10682 |
// happens the uses are usually next to each other in the list. |
--- |
| 10683 |
// To help reduce the number of CSE recomputations, process all |
--- |
10683 |
// To help reduce the number of CSE recomputations, process all |
--- |
| 10684 |
// the uses of this user that we can find this way. |
--- |
10684 |
// the uses of this user that we can find this way. |
--- |
| 10685 |
do { |
--- |
10685 |
do { |
--- |
| 10686 |
SDUse &Use = UI.getUse(); |
2 |
10686 |
SDUse &Use = UI.getUse(); |
2 |
| 10687 |
++UI; |
2 |
10687 |
++UI; |
2 |
| 10688 |
Use.set(To); |
2 |
10688 |
Use.set(To); |
2 |
| 10689 |
if (To->isDivergent() != From->isDivergent()) |
2 |
10689 |
if (To->isDivergent() != From->isDivergent()) |
2 |
| 10690 |
updateDivergence(User); |
0 |
10690 |
updateDivergence(User); |
0 |
| 10691 |
} while (UI != UE && *UI == User); |
2 |
10691 |
} while (UI != UE && *UI == User); |
2 |
| 10692 |
// Now that we have modified User, add it back to the CSE maps. If it |
--- |
10692 |
// Now that we have modified User, add it back to the CSE maps. If it |
--- |
| 10693 |
// already exists there, recursively merge the results together. |
--- |
10693 |
// already exists there, recursively merge the results together. |
--- |
| 10694 |
AddModifiedNodeToCSEMaps(User); |
2 |
10694 |
AddModifiedNodeToCSEMaps(User); |
2 |
| 10695 |
} |
--- |
10695 |
} |
--- |
| 10696 |
|
--- |
10696 |
|
--- |
| 10697 |
// If we just RAUW'd the root, take note. |
--- |
10697 |
// If we just RAUW'd the root, take note. |
--- |
| 10698 |
if (FromN == getRoot()) |
2 |
10698 |
if (FromN == getRoot()) |
2 |
| 10699 |
setRoot(To); |
0 |
10699 |
setRoot(To); |
0 |
| 10700 |
} |
2 |
10700 |
} |
2 |
| 10701 |
|
--- |
10701 |
|
--- |
| 10702 |
/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. |
--- |
10702 |
/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. |
--- |
| 10703 |
/// This can cause recursive merging of nodes in the DAG. |
--- |
10703 |
/// This can cause recursive merging of nodes in the DAG. |
--- |
| 10704 |
/// |
--- |
10704 |
/// |
--- |
| 10705 |
/// This version assumes that for each value of From, there is a |
--- |
10705 |
/// This version assumes that for each value of From, there is a |
--- |
| 10706 |
/// corresponding value in To in the same position with the same type. |
--- |
10706 |
/// corresponding value in To in the same position with the same type. |
--- |
| 10707 |
/// |
--- |
10707 |
/// |
--- |
| 10708 |
void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) { |
1 |
10708 |
void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) { |
1 |
| 10709 |
#ifndef NDEBUG |
--- |
10709 |
#ifndef NDEBUG |
--- |
| 10710 |
for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) |
2 |
10710 |
for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) |
2 |
| 10711 |
assert((!From->hasAnyUseOfValue(i) || |
1 |
10711 |
assert((!From->hasAnyUseOfValue(i) || |
1 |
| 10712 |
From->getValueType(i) == To->getValueType(i)) && |
--- |
10712 |
From->getValueType(i) == To->getValueType(i)) && |
--- |
| 10713 |
"Cannot use this version of ReplaceAllUsesWith!"); |
--- |
10713 |
"Cannot use this version of ReplaceAllUsesWith!"); |
--- |
| 10714 |
#endif |
--- |
10714 |
#endif |
--- |
| 10715 |
|
--- |
10715 |
|
--- |
| 10716 |
// Handle the trivial case. |
--- |
10716 |
// Handle the trivial case. |
--- |
| 10717 |
if (From == To) |
1 |
10717 |
if (From == To) |
1 |
| 10718 |
return; |
0 |
10718 |
return; |
0 |
| 10719 |
|
--- |
10719 |
|
--- |
| 10720 |
// Preserve Debug Info. Only do this if there's a use. |
--- |
10720 |
// Preserve Debug Info. Only do this if there's a use. |
--- |
| 10721 |
for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) |
2 |
10721 |
for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) |
2 |
| 10722 |
if (From->hasAnyUseOfValue(i)) { |
1 |
10722 |
if (From->hasAnyUseOfValue(i)) { |
1 |
| 10723 |
assert((i < To->getNumValues()) && "Invalid To location"); |
1 |
10723 |
assert((i < To->getNumValues()) && "Invalid To location"); |
1 |
| 10724 |
transferDbgValues(SDValue(From, i), SDValue(To, i)); |
1 |
10724 |
transferDbgValues(SDValue(From, i), SDValue(To, i)); |
1 |
| 10725 |
} |
--- |
10725 |
} |
--- |
| 10726 |
// Preserve extra info. |
--- |
10726 |
// Preserve extra info. |
--- |
| 10727 |
copyExtraInfo(From, To); |
1 |
10727 |
copyExtraInfo(From, To); |
1 |
| 10728 |
|
--- |
10728 |
|
--- |
| 10729 |
// Iterate over just the existing users of From. See the comments in |
--- |
10729 |
// Iterate over just the existing users of From. See the comments in |
--- |
| 10730 |
// the ReplaceAllUsesWith above. |
--- |
10730 |
// the ReplaceAllUsesWith above. |
--- |
| 10731 |
SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); |
1 |
10731 |
SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); |
1 |
| 10732 |
RAUWUpdateListener Listener(*this, UI, UE); |
1 |
10732 |
RAUWUpdateListener Listener(*this, UI, UE); |
1 |
| 10733 |
while (UI != UE) { |
2 |
10733 |
while (UI != UE) { |
2 |
| 10734 |
SDNode *User = *UI; |
1 |
10734 |
SDNode *User = *UI; |
1 |
| 10735 |
|
--- |
10735 |
|
--- |
| 10736 |
// This node is about to morph, remove its old self from the CSE maps. |
--- |
10736 |
// This node is about to morph, remove its old self from the CSE maps. |
--- |
| 10737 |
RemoveNodeFromCSEMaps(User); |
1 |
10737 |
RemoveNodeFromCSEMaps(User); |
1 |
| 10738 |
|
--- |
10738 |
|
--- |
| 10739 |
// A user can appear in a use list multiple times, and when this |
--- |
10739 |
// A user can appear in a use list multiple times, and when this |
--- |
| 10740 |
// happens the uses are usually next to each other in the list. |
--- |
10740 |
// happens the uses are usually next to each other in the list. |
--- |
| 10741 |
// To help reduce the number of CSE recomputations, process all |
--- |
10741 |
// To help reduce the number of CSE recomputations, process all |
--- |
| 10742 |
// the uses of this user that we can find this way. |
--- |
10742 |
// the uses of this user that we can find this way. |
--- |
| 10743 |
do { |
--- |
10743 |
do { |
--- |
| 10744 |
SDUse &Use = UI.getUse(); |
1 |
10744 |
SDUse &Use = UI.getUse(); |
1 |
| 10745 |
++UI; |
1 |
10745 |
++UI; |
1 |
| 10746 |
Use.setNode(To); |
1 |
10746 |
Use.setNode(To); |
1 |
| 10747 |
if (To->isDivergent() != From->isDivergent()) |
1 |
10747 |
if (To->isDivergent() != From->isDivergent()) |
1 |
| 10748 |
updateDivergence(User); |
0 |
10748 |
updateDivergence(User); |
0 |
| 10749 |
} while (UI != UE && *UI == User); |
1 |
10749 |
} while (UI != UE && *UI == User); |
1 |
| 10750 |
|
--- |
10750 |
|
--- |
| 10751 |
// Now that we have modified User, add it back to the CSE maps. If it |
--- |
10751 |
// Now that we have modified User, add it back to the CSE maps. If it |
--- |
| 10752 |
// already exists there, recursively merge the results together. |
--- |
10752 |
// already exists there, recursively merge the results together. |
--- |
| 10753 |
AddModifiedNodeToCSEMaps(User); |
1 |
10753 |
AddModifiedNodeToCSEMaps(User); |
1 |
| 10754 |
} |
--- |
10754 |
} |
--- |
| 10755 |
|
--- |
10755 |
|
--- |
| 10756 |
// If we just RAUW'd the root, take note. |
--- |
10756 |
// If we just RAUW'd the root, take note. |
--- |
| 10757 |
if (From == getRoot().getNode()) |
1 |
10757 |
if (From == getRoot().getNode()) |
1 |
| 10758 |
setRoot(SDValue(To, getRoot().getResNo())); |
0 |
10758 |
setRoot(SDValue(To, getRoot().getResNo())); |
0 |
| 10759 |
} |
1 |
10759 |
} |
1 |
| 10760 |
|
--- |
10760 |
|
--- |
| 10761 |
/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. |
--- |
10761 |
/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. |
--- |
| 10762 |
/// This can cause recursive merging of nodes in the DAG. |
--- |
10762 |
/// This can cause recursive merging of nodes in the DAG. |
--- |
| 10763 |
/// |
--- |
10763 |
/// |
--- |
| 10764 |
/// This version can replace From with any result values. To must match the |
--- |
10764 |
/// This version can replace From with any result values. To must match the |
--- |
| 10765 |
/// number and types of values returned by From. |
--- |
10765 |
/// number and types of values returned by From. |
--- |
| 10766 |
void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) { |
0 |
10766 |
void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) { |
0 |
| 10767 |
if (From->getNumValues() == 1) // Handle the simple case efficiently. |
0 |
10767 |
if (From->getNumValues() == 1) // Handle the simple case efficiently. |
0 |
| 10768 |
return ReplaceAllUsesWith(SDValue(From, 0), To[0]); |
0 |
10768 |
return ReplaceAllUsesWith(SDValue(From, 0), To[0]); |
0 |
| 10769 |
|
--- |
10769 |
|
--- |
| 10770 |
for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) { |
0 |
10770 |
for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) { |
0 |
| 10771 |
// Preserve Debug Info. |
--- |
10771 |
// Preserve Debug Info. |
--- |
| 10772 |
transferDbgValues(SDValue(From, i), To[i]); |
0 |
10772 |
transferDbgValues(SDValue(From, i), To[i]); |
0 |
| 10773 |
// Preserve extra info. |
--- |
10773 |
// Preserve extra info. |
--- |
| 10774 |
copyExtraInfo(From, To[i].getNode()); |
0 |
10774 |
copyExtraInfo(From, To[i].getNode()); |
0 |
| 10775 |
} |
--- |
10775 |
} |
--- |
| 10776 |
|
--- |
10776 |
|
--- |
| 10777 |
// Iterate over just the existing users of From. See the comments in |
--- |
10777 |
// Iterate over just the existing users of From. See the comments in |
--- |
| 10778 |
// the ReplaceAllUsesWith above. |
--- |
10778 |
// the ReplaceAllUsesWith above. |
--- |
| 10779 |
SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); |
0 |
10779 |
SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); |
0 |
| 10780 |
RAUWUpdateListener Listener(*this, UI, UE); |
0 |
10780 |
RAUWUpdateListener Listener(*this, UI, UE); |
0 |
| 10781 |
while (UI != UE) { |
0 |
10781 |
while (UI != UE) { |
0 |
| 10782 |
SDNode *User = *UI; |
0 |
10782 |
SDNode *User = *UI; |
0 |
| 10783 |
|
--- |
10783 |
|
--- |
| 10784 |
// This node is about to morph, remove its old self from the CSE maps. |
--- |
10784 |
// This node is about to morph, remove its old self from the CSE maps. |
--- |
| 10785 |
RemoveNodeFromCSEMaps(User); |
0 |
10785 |
RemoveNodeFromCSEMaps(User); |
0 |
| 10786 |
|
--- |
10786 |
|
--- |
| 10787 |
// A user can appear in a use list multiple times, and when this happens the |
--- |
10787 |
// A user can appear in a use list multiple times, and when this happens the |
--- |
| 10788 |
// uses are usually next to each other in the list. To help reduce the |
--- |
10788 |
// uses are usually next to each other in the list. To help reduce the |
--- |
| 10789 |
// number of CSE and divergence recomputations, process all the uses of this |
--- |
10789 |
// number of CSE and divergence recomputations, process all the uses of this |
--- |
| 10790 |
// user that we can find this way. |
--- |
10790 |
// user that we can find this way. |
--- |
| 10791 |
bool To_IsDivergent = false; |
0 |
10791 |
bool To_IsDivergent = false; |
0 |
| 10792 |
do { |
--- |
10792 |
do { |
--- |
| 10793 |
SDUse &Use = UI.getUse(); |
0 |
10793 |
SDUse &Use = UI.getUse(); |
0 |
| 10794 |
const SDValue &ToOp = To[Use.getResNo()]; |
0 |
10794 |
const SDValue &ToOp = To[Use.getResNo()]; |
0 |
| 10795 |
++UI; |
0 |
10795 |
++UI; |
0 |
| 10796 |
Use.set(ToOp); |
0 |
10796 |
Use.set(ToOp); |
0 |
| 10797 |
To_IsDivergent |= ToOp->isDivergent(); |
0 |
10797 |
To_IsDivergent |= ToOp->isDivergent(); |
0 |
| 10798 |
} while (UI != UE && *UI == User); |
0 |
10798 |
} while (UI != UE && *UI == User); |
0 |
| 10799 |
|
--- |
10799 |
|
--- |
| 10800 |
if (To_IsDivergent != From->isDivergent()) |
0 |
10800 |
if (To_IsDivergent != From->isDivergent()) |
0 |
| 10801 |
updateDivergence(User); |
0 |
10801 |
updateDivergence(User); |
0 |
| 10802 |
|
--- |
10802 |
|
--- |
| 10803 |
// Now that we have modified User, add it back to the CSE maps. If it |
--- |
10803 |
// Now that we have modified User, add it back to the CSE maps. If it |
--- |
| 10804 |
// already exists there, recursively merge the results together. |
--- |
10804 |
// already exists there, recursively merge the results together. |
--- |
| 10805 |
AddModifiedNodeToCSEMaps(User); |
0 |
10805 |
AddModifiedNodeToCSEMaps(User); |
0 |
| 10806 |
} |
--- |
10806 |
} |
--- |
| 10807 |
|
--- |
10807 |
|
--- |
| 10808 |
// If we just RAUW'd the root, take note. |
--- |
10808 |
// If we just RAUW'd the root, take note. |
--- |
| 10809 |
if (From == getRoot().getNode()) |
0 |
10809 |
if (From == getRoot().getNode()) |
0 |
| 10810 |
setRoot(SDValue(To[getRoot().getResNo()])); |
0 |
10810 |
setRoot(SDValue(To[getRoot().getResNo()])); |
0 |
| 10811 |
} |
0 |
10811 |
} |
0 |
| 10812 |
|
--- |
10812 |
|
--- |
| 10813 |
/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving |
--- |
10813 |
/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving |
--- |
| 10814 |
/// uses of other values produced by From.getNode() alone. The Deleted |
--- |
10814 |
/// uses of other values produced by From.getNode() alone. The Deleted |
--- |
| 10815 |
/// vector is handled the same way as for ReplaceAllUsesWith. |
--- |
10815 |
/// vector is handled the same way as for ReplaceAllUsesWith. |
--- |
| 10816 |
void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){ |
2 |
10816 |
void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){ |
2 |
| 10817 |
// Handle the really simple, really trivial case efficiently. |
--- |
10817 |
// Handle the really simple, really trivial case efficiently. |
--- |
| 10818 |
if (From == To) return; |
4 |
10818 |
if (From == To) return; |
4 |
| 10819 |
|
--- |
10819 |
|
--- |
| 10820 |
// Handle the simple, trivial, case efficiently. |
--- |
10820 |
// Handle the simple, trivial, case efficiently. |
--- |
| 10821 |
if (From.getNode()->getNumValues() == 1) { |
2 |
10821 |
if (From.getNode()->getNumValues() == 1) { |
2 |
| 10822 |
ReplaceAllUsesWith(From, To); |
2 |
10822 |
ReplaceAllUsesWith(From, To); |
2 |
| 10823 |
return; |
2 |
10823 |
return; |
2 |
| 10824 |
} |
--- |
10824 |
} |
--- |
| 10825 |
|
--- |
10825 |
|
--- |
| 10826 |
// Preserve Debug Info. |
--- |
10826 |
// Preserve Debug Info. |
--- |
| 10827 |
transferDbgValues(From, To); |
0 |
10827 |
transferDbgValues(From, To); |
0 |
| 10828 |
copyExtraInfo(From.getNode(), To.getNode()); |
0 |
10828 |
copyExtraInfo(From.getNode(), To.getNode()); |
0 |
| 10829 |
|
--- |
10829 |
|
--- |
| 10830 |
// Iterate over just the existing users of From. See the comments in |
--- |
10830 |
// Iterate over just the existing users of From. See the comments in |
--- |
| 10831 |
// the ReplaceAllUsesWith above. |
--- |
10831 |
// the ReplaceAllUsesWith above. |
--- |
| 10832 |
SDNode::use_iterator UI = From.getNode()->use_begin(), |
0 |
10832 |
SDNode::use_iterator UI = From.getNode()->use_begin(), |
0 |
| 10833 |
UE = From.getNode()->use_end(); |
0 |
10833 |
UE = From.getNode()->use_end(); |
0 |
| 10834 |
RAUWUpdateListener Listener(*this, UI, UE); |
0 |
10834 |
RAUWUpdateListener Listener(*this, UI, UE); |
0 |
| 10835 |
while (UI != UE) { |
0 |
10835 |
while (UI != UE) { |
0 |
| 10836 |
SDNode *User = *UI; |
0 |
10836 |
SDNode *User = *UI; |
0 |
| 10837 |
bool UserRemovedFromCSEMaps = false; |
0 |
10837 |
bool UserRemovedFromCSEMaps = false; |
0 |
| 10838 |
|
--- |
10838 |
|
--- |
| 10839 |
// A user can appear in a use list multiple times, and when this |
--- |
10839 |
// A user can appear in a use list multiple times, and when this |
--- |
| 10840 |
// happens the uses are usually next to each other in the list. |
--- |
10840 |
// happens the uses are usually next to each other in the list. |
--- |
| 10841 |
// To help reduce the number of CSE recomputations, process all |
--- |
10841 |
// To help reduce the number of CSE recomputations, process all |
--- |
| 10842 |
// the uses of this user that we can find this way. |
--- |
10842 |
// the uses of this user that we can find this way. |
--- |
| 10843 |
do { |
--- |
10843 |
do { |
--- |
| 10844 |
SDUse &Use = UI.getUse(); |
0 |
10844 |
SDUse &Use = UI.getUse(); |
0 |
| 10845 |
|
--- |
10845 |
|
--- |
| 10846 |
// Skip uses of different values from the same node. |
--- |
10846 |
// Skip uses of different values from the same node. |
--- |
| 10847 |
if (Use.getResNo() != From.getResNo()) { |
0 |
10847 |
if (Use.getResNo() != From.getResNo()) { |
0 |
| 10848 |
++UI; |
0 |
10848 |
++UI; |
0 |
| 10849 |
continue; |
0 |
10849 |
continue; |
0 |
| 10850 |
} |
--- |
10850 |
} |
--- |
| 10851 |
|
--- |
10851 |
|
--- |
| 10852 |
// If this node hasn't been modified yet, it's still in the CSE maps, |
--- |
10852 |
// If this node hasn't been modified yet, it's still in the CSE maps, |
--- |
| 10853 |
// so remove its old self from the CSE maps. |
--- |
10853 |
// so remove its old self from the CSE maps. |
--- |
| 10854 |
if (!UserRemovedFromCSEMaps) { |
0 |
10854 |
if (!UserRemovedFromCSEMaps) { |
0 |
| 10855 |
RemoveNodeFromCSEMaps(User); |
0 |
10855 |
RemoveNodeFromCSEMaps(User); |
0 |
| 10856 |
UserRemovedFromCSEMaps = true; |
0 |
10856 |
UserRemovedFromCSEMaps = true; |
0 |
| 10857 |
} |
--- |
10857 |
} |
--- |
| 10858 |
|
--- |
10858 |
|
--- |
| 10859 |
++UI; |
0 |
10859 |
++UI; |
0 |
| 10860 |
Use.set(To); |
0 |
10860 |
Use.set(To); |
0 |
| 10861 |
if (To->isDivergent() != From->isDivergent()) |
0 |
10861 |
if (To->isDivergent() != From->isDivergent()) |
0 |
| 10862 |
updateDivergence(User); |
0 |
10862 |
updateDivergence(User); |
0 |
| 10863 |
} while (UI != UE && *UI == User); |
0 |
10863 |
} while (UI != UE && *UI == User); |
0 |
| 10864 |
// We are iterating over all uses of the From node, so if a use |
--- |
10864 |
// We are iterating over all uses of the From node, so if a use |
--- |
| 10865 |
// doesn't use the specific value, no changes are made. |
--- |
10865 |
// doesn't use the specific value, no changes are made. |
--- |
| 10866 |
if (!UserRemovedFromCSEMaps) |
0 |
10866 |
if (!UserRemovedFromCSEMaps) |
0 |
| 10867 |
continue; |
0 |
10867 |
continue; |
0 |
| 10868 |
|
--- |
10868 |
|
--- |
| 10869 |
// Now that we have modified User, add it back to the CSE maps. If it |
--- |
10869 |
// Now that we have modified User, add it back to the CSE maps. If it |
--- |
| 10870 |
// already exists there, recursively merge the results together. |
--- |
10870 |
// already exists there, recursively merge the results together. |
--- |
| 10871 |
AddModifiedNodeToCSEMaps(User); |
0 |
10871 |
AddModifiedNodeToCSEMaps(User); |
0 |
| 10872 |
} |
--- |
10872 |
} |
--- |
| 10873 |
|
--- |
10873 |
|
--- |
| 10874 |
// If we just RAUW'd the root, take note. |
--- |
10874 |
// If we just RAUW'd the root, take note. |
--- |
| 10875 |
if (From == getRoot()) |
0 |
10875 |
if (From == getRoot()) |
0 |
| 10876 |
setRoot(To); |
0 |
10876 |
setRoot(To); |
0 |
| 10877 |
} |
0 |
10877 |
} |
0 |
| 10878 |
|
--- |
10878 |
|
--- |
| 10879 |
namespace { |
--- |
10879 |
namespace { |
--- |
| 10880 |
|
--- |
10880 |
|
--- |
| 10881 |
/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith |
--- |
10881 |
/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith |
--- |
| 10882 |
/// to record information about a use. |
--- |
10882 |
/// to record information about a use. |
--- |
| 10883 |
struct UseMemo { |
--- |
10883 |
struct UseMemo { |
--- |
| 10884 |
SDNode *User; |
--- |
10884 |
SDNode *User; |
--- |
| 10885 |
unsigned Index; |
--- |
10885 |
unsigned Index; |
--- |
| 10886 |
SDUse *Use; |
--- |
10886 |
SDUse *Use; |
--- |
| 10887 |
}; |
--- |
10887 |
}; |
--- |
| 10888 |
|
--- |
10888 |
|
--- |
| 10889 |
/// operator< - Sort Memos by User. |
--- |
10889 |
/// operator< - Sort Memos by User. |
--- |
| 10890 |
bool operator<(const UseMemo &L, const UseMemo &R) { |
0 |
10890 |
bool operator<(const UseMemo &L, const UseMemo &R) { |
0 |
| 10891 |
return (intptr_t)L.User < (intptr_t)R.User; |
0 |
10891 |
return (intptr_t)L.User < (intptr_t)R.User; |
0 |
| 10892 |
} |
--- |
10892 |
} |
--- |
| 10893 |
|
--- |
10893 |
|
--- |
| 10894 |
/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node |
--- |
10894 |
/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node |
--- |
| 10895 |
/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that |
--- |
10895 |
/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that |
--- |
| 10896 |
/// the node already has been taken care of recursively. |
--- |
10896 |
/// the node already has been taken care of recursively. |
--- |
| 10897 |
class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener { |
--- |
10897 |
class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener { |
--- |
| 10898 |
SmallVector &Uses; |
--- |
10898 |
SmallVector &Uses; |
--- |
| 10899 |
|
--- |
10899 |
|
--- |
| 10900 |
void NodeDeleted(SDNode *N, SDNode *E) override { |
0 |
10900 |
void NodeDeleted(SDNode *N, SDNode *E) override { |
0 |
| 10901 |
for (UseMemo &Memo : Uses) |
0 |
10901 |
for (UseMemo &Memo : Uses) |
0 |
| 10902 |
if (Memo.User == N) |
0 |
10902 |
if (Memo.User == N) |
0 |
| 10903 |
Memo.User = nullptr; |
0 |
10903 |
Memo.User = nullptr; |
0 |
| 10904 |
} |
0 |
10904 |
} |
0 |
| 10905 |
|
--- |
10905 |
|
--- |
| 10906 |
public: |
--- |
10906 |
public: |
--- |
| 10907 |
RAUOVWUpdateListener(SelectionDAG &d, SmallVector &uses) |
0 |
10907 |
RAUOVWUpdateListener(SelectionDAG &d, SmallVector &uses) |
0 |
| 10908 |
: SelectionDAG::DAGUpdateListener(d), Uses(uses) {} |
0 |
10908 |
: SelectionDAG::DAGUpdateListener(d), Uses(uses) {} |
0 |
| 10909 |
}; |
--- |
10909 |
}; |
--- |
| 10910 |
|
--- |
10910 |
|
--- |
| 10911 |
} // end anonymous namespace |
--- |
10911 |
} // end anonymous namespace |
--- |
| 10912 |
|
--- |
10912 |
|
--- |
| 10913 |
bool SelectionDAG::calculateDivergence(SDNode *N) { |
1 |
10913 |
bool SelectionDAG::calculateDivergence(SDNode *N) { |
1 |
| 10914 |
if (TLI->isSDNodeAlwaysUniform(N)) { |
1 |
10914 |
if (TLI->isSDNodeAlwaysUniform(N)) { |
1 |
| 10915 |
assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) && |
0 |
10915 |
assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) && |
0 |
| 10916 |
"Conflicting divergence information!"); |
--- |
10916 |
"Conflicting divergence information!"); |
--- |
| 10917 |
return false; |
0 |
10917 |
return false; |
0 |
| 10918 |
} |
--- |
10918 |
} |
--- |
| 10919 |
if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA)) |
1 |
10919 |
if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA)) |
1 |
| 10920 |
return true; |
0 |
10920 |
return true; |
0 |
| 10921 |
for (const auto &Op : N->ops()) { |
4 |
10921 |
for (const auto &Op : N->ops()) { |
4 |
| 10922 |
if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent()) |
3 |
10922 |
if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent()) |
3 |
| 10923 |
return true; |
0 |
10923 |
return true; |
0 |
| 10924 |
} |
--- |
10924 |
} |
--- |
| 10925 |
return false; |
1 |
10925 |
return false; |
1 |
| 10926 |
} |
--- |
10926 |
} |
--- |
| 10927 |
|
--- |
10927 |
|
--- |
| 10928 |
void SelectionDAG::updateDivergence(SDNode *N) { |
1 |
10928 |
void SelectionDAG::updateDivergence(SDNode *N) { |
1 |
| 10929 |
SmallVector Worklist(1, N); |
1 |
10929 |
SmallVector Worklist(1, N); |
1 |
| 10930 |
do { |
--- |
10930 |
do { |
--- |
| 10931 |
N = Worklist.pop_back_val(); |
1 |
10931 |
N = Worklist.pop_back_val(); |
1 |
| 10932 |
bool IsDivergent = calculateDivergence(N); |
1 |
10932 |
bool IsDivergent = calculateDivergence(N); |
1 |
| 10933 |
if (N->SDNodeBits.IsDivergent != IsDivergent) { |
1 |
10933 |
if (N->SDNodeBits.IsDivergent != IsDivergent) { |
1 |
| 10934 |
N->SDNodeBits.IsDivergent = IsDivergent; |
0 |
10934 |
N->SDNodeBits.IsDivergent = IsDivergent; |
0 |
| 10935 |
llvm::append_range(Worklist, N->uses()); |
0 |
10935 |
llvm::append_range(Worklist, N->uses()); |
0 |
| 10936 |
} |
--- |
10936 |
} |
--- |
| 10937 |
} while (!Worklist.empty()); |
1 |
10937 |
} while (!Worklist.empty()); |
1 |
| 10938 |
} |
1 |
10938 |
} |
1 |
| 10939 |
|
--- |
10939 |
|
--- |
| 10940 |
void SelectionDAG::CreateTopologicalOrder(std::vector &Order) { |
0 |
10940 |
void SelectionDAG::CreateTopologicalOrder(std::vector &Order) { |
0 |
| 10941 |
DenseMap Degree; |
0 |
10941 |
DenseMap Degree; |
0 |
| 10942 |
Order.reserve(AllNodes.size()); |
0 |
10942 |
Order.reserve(AllNodes.size()); |
0 |
| 10943 |
for (auto &N : allnodes()) { |
0 |
10943 |
for (auto &N : allnodes()) { |
0 |
| 10944 |
unsigned NOps = N.getNumOperands(); |
0 |
10944 |
unsigned NOps = N.getNumOperands(); |
0 |
| 10945 |
Degree[&N] = NOps; |
0 |
10945 |
Degree[&N] = NOps; |
0 |
| 10946 |
if (0 == NOps) |
0 |
10946 |
if (0 == NOps) |
0 |
| 10947 |
Order.push_back(&N); |
0 |
10947 |
Order.push_back(&N); |
0 |
| 10948 |
} |
--- |
10948 |
} |
--- |
| 10949 |
for (size_t I = 0; I != Order.size(); ++I) { |
0 |
10949 |
for (size_t I = 0; I != Order.size(); ++I) { |
0 |
| 10950 |
SDNode *N = Order[I]; |
0 |
10950 |
SDNode *N = Order[I]; |
0 |
| 10951 |
for (auto *U : N->uses()) { |
0 |
10951 |
for (auto *U : N->uses()) { |
0 |
| 10952 |
unsigned &UnsortedOps = Degree[U]; |
0 |
10952 |
unsigned &UnsortedOps = Degree[U]; |
0 |
| 10953 |
if (0 == --UnsortedOps) |
0 |
10953 |
if (0 == --UnsortedOps) |
0 |
| 10954 |
Order.push_back(U); |
0 |
10954 |
Order.push_back(U); |
0 |
| 10955 |
} |
--- |
10955 |
} |
--- |
| 10956 |
} |
--- |
10956 |
} |
--- |
| 10957 |
} |
0 |
10957 |
} |
0 |
| 10958 |
|
--- |
10958 |
|
--- |
| 10959 |
#ifndef NDEBUG |
--- |
10959 |
#ifndef NDEBUG |
--- |
| 10960 |
void SelectionDAG::VerifyDAGDivergence() { |
0 |
10960 |
void SelectionDAG::VerifyDAGDivergence() { |
0 |
| 10961 |
std::vector TopoOrder; |
0 |
10961 |
std::vector TopoOrder; |
0 |
| 10962 |
CreateTopologicalOrder(TopoOrder); |
0 |
10962 |
CreateTopologicalOrder(TopoOrder); |
0 |
| 10963 |
for (auto *N : TopoOrder) { |
0 |
10963 |
for (auto *N : TopoOrder) { |
0 |
| 10964 |
assert(calculateDivergence(N) == N->isDivergent() && |
0 |
10964 |
assert(calculateDivergence(N) == N->isDivergent() && |
0 |
| 10965 |
"Divergence bit inconsistency detected"); |
--- |
10965 |
"Divergence bit inconsistency detected"); |
--- |
| 10966 |
} |
--- |
10966 |
} |
--- |
| 10967 |
} |
0 |
10967 |
} |
0 |
| 10968 |
#endif |
--- |
10968 |
#endif |
--- |
| 10969 |
|
--- |
10969 |
|
--- |
| 10970 |
/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving |
--- |
10970 |
/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving |
--- |
| 10971 |
/// uses of other values produced by From.getNode() alone. The same value |
--- |
10971 |
/// uses of other values produced by From.getNode() alone. The same value |
--- |
| 10972 |
/// may appear in both the From and To list. The Deleted vector is |
--- |
10972 |
/// may appear in both the From and To list. The Deleted vector is |
--- |
| 10973 |
/// handled the same way as for ReplaceAllUsesWith. |
--- |
10973 |
/// handled the same way as for ReplaceAllUsesWith. |
--- |
| 10974 |
void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, |
0 |
10974 |
void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, |
0 |
| 10975 |
const SDValue *To, |
--- |
10975 |
const SDValue *To, |
--- |
| 10976 |
unsigned Num){ |
--- |
10976 |
unsigned Num){ |
--- |
| 10977 |
// Handle the simple, trivial case efficiently. |
--- |
10977 |
// Handle the simple, trivial case efficiently. |
--- |
| 10978 |
if (Num == 1) |
0 |
10978 |
if (Num == 1) |
0 |
| 10979 |
return ReplaceAllUsesOfValueWith(*From, *To); |
0 |
10979 |
return ReplaceAllUsesOfValueWith(*From, *To); |
0 |
| 10980 |
|
--- |
10980 |
|
--- |
| 10981 |
transferDbgValues(*From, *To); |
0 |
10981 |
transferDbgValues(*From, *To); |
0 |
| 10982 |
copyExtraInfo(From->getNode(), To->getNode()); |
0 |
10982 |
copyExtraInfo(From->getNode(), To->getNode()); |
0 |
| 10983 |
|
--- |
10983 |
|
--- |
| 10984 |
// Read up all the uses and make records of them. This helps |
--- |
10984 |
// Read up all the uses and make records of them. This helps |
--- |
| 10985 |
// processing new uses that are introduced during the |
--- |
10985 |
// processing new uses that are introduced during the |
--- |
| 10986 |
// replacement process. |
--- |
10986 |
// replacement process. |
--- |
| 10987 |
SmallVector Uses; |
0 |
10987 |
SmallVector Uses; |
0 |
| 10988 |
for (unsigned i = 0; i != Num; ++i) { |
0 |
10988 |
for (unsigned i = 0; i != Num; ++i) { |
0 |
| 10989 |
unsigned FromResNo = From[i].getResNo(); |
0 |
10989 |
unsigned FromResNo = From[i].getResNo(); |
0 |
| 10990 |
SDNode *FromNode = From[i].getNode(); |
0 |
10990 |
SDNode *FromNode = From[i].getNode(); |
0 |
| 10991 |
for (SDNode::use_iterator UI = FromNode->use_begin(), |
0 |
10991 |
for (SDNode::use_iterator UI = FromNode->use_begin(), |
0 |
| 10992 |
E = FromNode->use_end(); UI != E; ++UI) { |
0 |
10992 |
E = FromNode->use_end(); UI != E; ++UI) { |
0 |
| 10993 |
SDUse &Use = UI.getUse(); |
0 |
10993 |
SDUse &Use = UI.getUse(); |
0 |
| 10994 |
if (Use.getResNo() == FromResNo) { |
0 |
10994 |
if (Use.getResNo() == FromResNo) { |
0 |
| 10995 |
UseMemo Memo = { *UI, i, &Use }; |
0 |
10995 |
UseMemo Memo = { *UI, i, &Use }; |
0 |
| 10996 |
Uses.push_back(Memo); |
0 |
10996 |
Uses.push_back(Memo); |
0 |
| 10997 |
} |
--- |
10997 |
} |
--- |
| 10998 |
} |
--- |
10998 |
} |
--- |
| 10999 |
} |
--- |
10999 |
} |
--- |
| 11000 |
|
--- |
11000 |
|
--- |
| 11001 |
// Sort the uses, so that all the uses from a given User are together. |
--- |
11001 |
// Sort the uses, so that all the uses from a given User are together. |
--- |
| 11002 |
llvm::sort(Uses); |
0 |
11002 |
llvm::sort(Uses); |
0 |
| 11003 |
RAUOVWUpdateListener Listener(*this, Uses); |
0 |
11003 |
RAUOVWUpdateListener Listener(*this, Uses); |
0 |
| 11004 |
|
--- |
11004 |
|
--- |
| 11005 |
for (unsigned UseIndex = 0, UseIndexEnd = Uses.size(); |
0 |
11005 |
for (unsigned UseIndex = 0, UseIndexEnd = Uses.size(); |
0 |
| 11006 |
UseIndex != UseIndexEnd; ) { |
0 |
11006 |
UseIndex != UseIndexEnd; ) { |
0 |
| 11007 |
// We know that this user uses some value of From. If it is the right |
--- |
11007 |
// We know that this user uses some value of From. If it is the right |
--- |
| 11008 |
// value, update it. |
--- |
11008 |
// value, update it. |
--- |
| 11009 |
SDNode *User = Uses[UseIndex].User; |
0 |
11009 |
SDNode *User = Uses[UseIndex].User; |
0 |
| 11010 |
// If the node has been deleted by recursive CSE updates when updating |
--- |
11010 |
// If the node has been deleted by recursive CSE updates when updating |
--- |
| 11011 |
// another node, then just skip this entry. |
--- |
11011 |
// another node, then just skip this entry. |
--- |
| 11012 |
if (User == nullptr) { |
0 |
11012 |
if (User == nullptr) { |
0 |
| 11013 |
++UseIndex; |
0 |
11013 |
++UseIndex; |
0 |
| 11014 |
continue; |
0 |
11014 |
continue; |
0 |
| 11015 |
} |
--- |
11015 |
} |
--- |
| 11016 |
|
--- |
11016 |
|
--- |
| 11017 |
// This node is about to morph, remove its old self from the CSE maps. |
--- |
11017 |
// This node is about to morph, remove its old self from the CSE maps. |
--- |
| 11018 |
RemoveNodeFromCSEMaps(User); |
0 |
11018 |
RemoveNodeFromCSEMaps(User); |
0 |
| 11019 |
|
--- |
11019 |
|
--- |
| 11020 |
// The Uses array is sorted, so all the uses for a given User |
--- |
11020 |
// The Uses array is sorted, so all the uses for a given User |
--- |
| 11021 |
// are next to each other in the list. |
--- |
11021 |
// are next to each other in the list. |
--- |
| 11022 |
// To help reduce the number of CSE recomputations, process all |
--- |
11022 |
// To help reduce the number of CSE recomputations, process all |
--- |
| 11023 |
// the uses of this user that we can find this way. |
--- |
11023 |
// the uses of this user that we can find this way. |
--- |
| 11024 |
do { |
--- |
11024 |
do { |
--- |
| 11025 |
unsigned i = Uses[UseIndex].Index; |
0 |
11025 |
unsigned i = Uses[UseIndex].Index; |
0 |
| 11026 |
SDUse &Use = *Uses[UseIndex].Use; |
0 |
11026 |
SDUse &Use = *Uses[UseIndex].Use; |
0 |
| 11027 |
++UseIndex; |
0 |
11027 |
++UseIndex; |
0 |
| 11028 |
|
--- |
11028 |
|
--- |
| 11029 |
Use.set(To[i]); |
0 |
11029 |
Use.set(To[i]); |
0 |
| 11030 |
} while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User); |
0 |
11030 |
} while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User); |
0 |
| 11031 |
|
--- |
11031 |
|
--- |
| 11032 |
// Now that we have modified User, add it back to the CSE maps. If it |
--- |
11032 |
// Now that we have modified User, add it back to the CSE maps. If it |
--- |
| 11033 |
// already exists there, recursively merge the results together. |
--- |
11033 |
// already exists there, recursively merge the results together. |
--- |
| 11034 |
AddModifiedNodeToCSEMaps(User); |
0 |
11034 |
AddModifiedNodeToCSEMaps(User); |
0 |
| 11035 |
} |
--- |
11035 |
} |
--- |
| 11036 |
} |
0 |
11036 |
} |
0 |
| 11037 |
|
--- |
11037 |
|
--- |
| 11038 |
/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG |
--- |
11038 |
/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG |
--- |
| 11039 |
/// based on their topological order. It returns the maximum id and a vector |
--- |
11039 |
/// based on their topological order. It returns the maximum id and a vector |
--- |
| 11040 |
/// of the SDNodes* in assigned order by reference. |
--- |
11040 |
/// of the SDNodes* in assigned order by reference. |
--- |
| 11041 |
unsigned SelectionDAG::AssignTopologicalOrder() { |
8 |
11041 |
unsigned SelectionDAG::AssignTopologicalOrder() { |
8 |
| 11042 |
unsigned DAGSize = 0; |
8 |
11042 |
unsigned DAGSize = 0; |
8 |
| 11043 |
|
--- |
11043 |
|
--- |
| 11044 |
// SortedPos tracks the progress of the algorithm. Nodes before it are |
--- |
11044 |
// SortedPos tracks the progress of the algorithm. Nodes before it are |
--- |
| 11045 |
// sorted, nodes after it are unsorted. When the algorithm completes |
--- |
11045 |
// sorted, nodes after it are unsorted. When the algorithm completes |
--- |
| 11046 |
// it is at the end of the list. |
--- |
11046 |
// it is at the end of the list. |
--- |
| 11047 |
allnodes_iterator SortedPos = allnodes_begin(); |
8 |
11047 |
allnodes_iterator SortedPos = allnodes_begin(); |
8 |
| 11048 |
|
--- |
11048 |
|
--- |
| 11049 |
// Visit all the nodes. Move nodes with no operands to the front of |
--- |
11049 |
// Visit all the nodes. Move nodes with no operands to the front of |
--- |
| 11050 |
// the list immediately. Annotate nodes that do have operands with their |
--- |
11050 |
// the list immediately. Annotate nodes that do have operands with their |
--- |
| 11051 |
// operand count. Before we do this, the Node Id fields of the nodes |
--- |
11051 |
// operand count. Before we do this, the Node Id fields of the nodes |
--- |
| 11052 |
// may contain arbitrary values. After, the Node Id fields for nodes |
--- |
11052 |
// may contain arbitrary values. After, the Node Id fields for nodes |
--- |
| 11053 |
// before SortedPos will contain the topological sort index, and the |
--- |
11053 |
// before SortedPos will contain the topological sort index, and the |
--- |
| 11054 |
// Node Id fields for nodes At SortedPos and after will contain the |
--- |
11054 |
// Node Id fields for nodes At SortedPos and after will contain the |
--- |
| 11055 |
// count of outstanding operands. |
--- |
11055 |
// count of outstanding operands. |
--- |
| 11056 |
for (SDNode &N : llvm::make_early_inc_range(allnodes())) { |
90 |
11056 |
for (SDNode &N : llvm::make_early_inc_range(allnodes())) { |
90 |
| 11057 |
checkForCycles(&N, this); |
82 |
11057 |
checkForCycles(&N, this); |
82 |
| 11058 |
unsigned Degree = N.getNumOperands(); |
82 |
11058 |
unsigned Degree = N.getNumOperands(); |
82 |
| 11059 |
if (Degree == 0) { |
82 |
11059 |
if (Degree == 0) { |
82 |
| 11060 |
// A node with no uses, add it to the result array immediately. |
--- |
11060 |
// A node with no uses, add it to the result array immediately. |
--- |
| 11061 |
N.setNodeId(DAGSize++); |
46 |
11061 |
N.setNodeId(DAGSize++); |
46 |
| 11062 |
allnodes_iterator Q(&N); |
46 |
11062 |
allnodes_iterator Q(&N); |
46 |
| 11063 |
if (Q != SortedPos) |
46 |
11063 |
if (Q != SortedPos) |
46 |
| 11064 |
SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); |
12 |
11064 |
SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); |
12 |
| 11065 |
assert(SortedPos != AllNodes.end() && "Overran node list"); |
46 |
11065 |
assert(SortedPos != AllNodes.end() && "Overran node list"); |
46 |
| 11066 |
++SortedPos; |
46 |
11066 |
++SortedPos; |
46 |
| 11067 |
} else { |
--- |
11067 |
} else { |
--- |
| 11068 |
// Temporarily use the Node Id as scratch space for the degree count. |
--- |
11068 |
// Temporarily use the Node Id as scratch space for the degree count. |
--- |
| 11069 |
N.setNodeId(Degree); |
36 |
11069 |
N.setNodeId(Degree); |
36 |
| 11070 |
} |
--- |
11070 |
} |
--- |
| 11071 |
} |
--- |
11071 |
} |
--- |
| 11072 |
|
--- |
11072 |
|
--- |
| 11073 |
// Visit all the nodes. As we iterate, move nodes into sorted order, |
--- |
11073 |
// Visit all the nodes. As we iterate, move nodes into sorted order, |
--- |
| 11074 |
// such that by the time the end is reached all nodes will be sorted. |
--- |
11074 |
// such that by the time the end is reached all nodes will be sorted. |
--- |
| 11075 |
for (SDNode &Node : allnodes()) { |
90 |
11075 |
for (SDNode &Node : allnodes()) { |
90 |
| 11076 |
SDNode *N = &Node; |
82 |
11076 |
SDNode *N = &Node; |
82 |
| 11077 |
checkForCycles(N, this); |
82 |
11077 |
checkForCycles(N, this); |
82 |
| 11078 |
// N is in sorted position, so all its uses have one less operand |
--- |
11078 |
// N is in sorted position, so all its uses have one less operand |
--- |
| 11079 |
// that needs to be sorted. |
--- |
11079 |
// that needs to be sorted. |
--- |
| 11080 |
for (SDNode *P : N->uses()) { |
188 |
11080 |
for (SDNode *P : N->uses()) { |
188 |
| 11081 |
unsigned Degree = P->getNodeId(); |
106 |
11081 |
unsigned Degree = P->getNodeId(); |
106 |
| 11082 |
assert(Degree != 0 && "Invalid node degree"); |
106 |
11082 |
assert(Degree != 0 && "Invalid node degree"); |
106 |
| 11083 |
--Degree; |
106 |
11083 |
--Degree; |
106 |
| 11084 |
if (Degree == 0) { |
106 |
11084 |
if (Degree == 0) { |
106 |
| 11085 |
// All of P's operands are sorted, so P may sorted now. |
--- |
11085 |
// All of P's operands are sorted, so P may sorted now. |
--- |
| 11086 |
P->setNodeId(DAGSize++); |
36 |
11086 |
P->setNodeId(DAGSize++); |
36 |
| 11087 |
if (P->getIterator() != SortedPos) |
36 |
11087 |
if (P->getIterator() != SortedPos) |
36 |
| 11088 |
SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); |
3 |
11088 |
SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); |
3 |
| 11089 |
assert(SortedPos != AllNodes.end() && "Overran node list"); |
36 |
11089 |
assert(SortedPos != AllNodes.end() && "Overran node list"); |
36 |
| 11090 |
++SortedPos; |
36 |
11090 |
++SortedPos; |
36 |
| 11091 |
} else { |
--- |
11091 |
} else { |
--- |
| 11092 |
// Update P's outstanding operand count. |
--- |
11092 |
// Update P's outstanding operand count. |
--- |
| 11093 |
P->setNodeId(Degree); |
70 |
11093 |
P->setNodeId(Degree); |
70 |
| 11094 |
} |
--- |
11094 |
} |
--- |
| 11095 |
} |
--- |
11095 |
} |
--- |
| 11096 |
if (Node.getIterator() == SortedPos) { |
82 |
11096 |
if (Node.getIterator() == SortedPos) { |
82 |
| 11097 |
#ifndef NDEBUG |
--- |
11097 |
#ifndef NDEBUG |
--- |
| 11098 |
allnodes_iterator I(N); |
0 |
11098 |
allnodes_iterator I(N); |
0 |
| 11099 |
SDNode *S = &*++I; |
0 |
11099 |
SDNode *S = &*++I; |
0 |
| 11100 |
dbgs() << "Overran sorted position:\n"; |
0 |
11100 |
dbgs() << "Overran sorted position:\n"; |
0 |
| 11101 |
S->dumprFull(this); dbgs() << "\n"; |
0 |
11101 |
S->dumprFull(this); dbgs() << "\n"; |
0 |
| 11102 |
dbgs() << "Checking if this is due to cycles\n"; |
0 |
11102 |
dbgs() << "Checking if this is due to cycles\n"; |
0 |
| 11103 |
checkForCycles(this, true); |
0 |
11103 |
checkForCycles(this, true); |
0 |
| 11104 |
#endif |
--- |
11104 |
#endif |
--- |
| 11105 |
llvm_unreachable(nullptr); |
0 |
11105 |
llvm_unreachable(nullptr); |
0 |
| 11106 |
} |
--- |
11106 |
} |
--- |
| 11107 |
} |
--- |
11107 |
} |
--- |
| 11108 |
|
--- |
11108 |
|
--- |
| 11109 |
assert(SortedPos == AllNodes.end() && |
8 |
11109 |
assert(SortedPos == AllNodes.end() && |
8 |
| 11110 |
"Topological sort incomplete!"); |
--- |
11110 |
"Topological sort incomplete!"); |
--- |
| 11111 |
assert(AllNodes.front().getOpcode() == ISD::EntryToken && |
8 |
11111 |
assert(AllNodes.front().getOpcode() == ISD::EntryToken && |
8 |
| 11112 |
"First node in topological sort is not the entry token!"); |
--- |
11112 |
"First node in topological sort is not the entry token!"); |
--- |
| 11113 |
assert(AllNodes.front().getNodeId() == 0 && |
8 |
11113 |
assert(AllNodes.front().getNodeId() == 0 && |
8 |
| 11114 |
"First node in topological sort has non-zero id!"); |
--- |
11114 |
"First node in topological sort has non-zero id!"); |
--- |
| 11115 |
assert(AllNodes.front().getNumOperands() == 0 && |
8 |
11115 |
assert(AllNodes.front().getNumOperands() == 0 && |
8 |
| 11116 |
"First node in topological sort has operands!"); |
--- |
11116 |
"First node in topological sort has operands!"); |
--- |
| 11117 |
assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && |
8 |
11117 |
assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && |
8 |
| 11118 |
"Last node in topologic sort has unexpected id!"); |
--- |
11118 |
"Last node in topologic sort has unexpected id!"); |
--- |
| 11119 |
assert(AllNodes.back().use_empty() && |
8 |
11119 |
assert(AllNodes.back().use_empty() && |
8 |
| 11120 |
"Last node in topologic sort has users!"); |
--- |
11120 |
"Last node in topologic sort has users!"); |
--- |
| 11121 |
assert(DAGSize == allnodes_size() && "Node count mismatch!"); |
8 |
11121 |
assert(DAGSize == allnodes_size() && "Node count mismatch!"); |
8 |
| 11122 |
return DAGSize; |
8 |
11122 |
return DAGSize; |
8 |
| 11123 |
} |
--- |
11123 |
} |
--- |
| 11124 |
|
--- |
11124 |
|
--- |
| 11125 |
/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the |
--- |
11125 |
/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the |
--- |
| 11126 |
/// value is produced by SD. |
--- |
11126 |
/// value is produced by SD. |
--- |
| 11127 |
void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) { |
0 |
11127 |
void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) { |
0 |
| 11128 |
for (SDNode *SD : DB->getSDNodes()) { |
0 |
11128 |
for (SDNode *SD : DB->getSDNodes()) { |
0 |
| 11129 |
if (!SD) |
0 |
11129 |
if (!SD) |
0 |
| 11130 |
continue; |
0 |
11130 |
continue; |
0 |
| 11131 |
assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue()); |
0 |
11131 |
assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue()); |
0 |
| 11132 |
SD->setHasDebugValue(true); |
0 |
11132 |
SD->setHasDebugValue(true); |
0 |
| 11133 |
} |
0 |
11133 |
} |
0 |
| 11134 |
DbgInfo->add(DB, isParameter); |
0 |
11134 |
DbgInfo->add(DB, isParameter); |
0 |
| 11135 |
} |
0 |
11135 |
} |
0 |
| 11136 |
|
--- |
11136 |
|
--- |
| 11137 |
void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(DB); } |
0 |
11137 |
void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(DB); } |
0 |
| 11138 |
|
--- |
11138 |
|
--- |
| 11139 |
SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain, |
0 |
11139 |
SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain, |
0 |
| 11140 |
SDValue NewMemOpChain) { |
--- |
11140 |
SDValue NewMemOpChain) { |
--- |
| 11141 |
assert(isa(NewMemOpChain) && "Expected a memop node"); |
0 |
11141 |
assert(isa(NewMemOpChain) && "Expected a memop node"); |
0 |
| 11142 |
assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT"); |
0 |
11142 |
assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT"); |
0 |
| 11143 |
// The new memory operation must have the same position as the old load in |
--- |
11143 |
// The new memory operation must have the same position as the old load in |
--- |
| 11144 |
// terms of memory dependency. Create a TokenFactor for the old load and new |
--- |
11144 |
// terms of memory dependency. Create a TokenFactor for the old load and new |
--- |
| 11145 |
// memory operation and update uses of the old load's output chain to use that |
--- |
11145 |
// memory operation and update uses of the old load's output chain to use that |
--- |
| 11146 |
// TokenFactor. |
--- |
11146 |
// TokenFactor. |
--- |
| 11147 |
if (OldChain == NewMemOpChain || OldChain.use_empty()) |
0 |
11147 |
if (OldChain == NewMemOpChain || OldChain.use_empty()) |
0 |
| 11148 |
return NewMemOpChain; |
0 |
11148 |
return NewMemOpChain; |
0 |
| 11149 |
|
--- |
11149 |
|
--- |
| 11150 |
SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other, |
0 |
11150 |
SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other, |
0 |
| 11151 |
OldChain, NewMemOpChain); |
--- |
11151 |
OldChain, NewMemOpChain); |
--- |
| 11152 |
ReplaceAllUsesOfValueWith(OldChain, TokenFactor); |
0 |
11152 |
ReplaceAllUsesOfValueWith(OldChain, TokenFactor); |
0 |
| 11153 |
UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewMemOpChain); |
0 |
11153 |
UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewMemOpChain); |
0 |
| 11154 |
return TokenFactor; |
0 |
11154 |
return TokenFactor; |
0 |
| 11155 |
} |
--- |
11155 |
} |
--- |
| 11156 |
|
--- |
11156 |
|
--- |
| 11157 |
SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, |
0 |
11157 |
SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, |
0 |
| 11158 |
SDValue NewMemOp) { |
--- |
11158 |
SDValue NewMemOp) { |
--- |
| 11159 |
assert(isa(NewMemOp.getNode()) && "Expected a memop node"); |
0 |
11159 |
assert(isa(NewMemOp.getNode()) && "Expected a memop node"); |
0 |
| 11160 |
SDValue OldChain = SDValue(OldLoad, 1); |
0 |
11160 |
SDValue OldChain = SDValue(OldLoad, 1); |
0 |
| 11161 |
SDValue NewMemOpChain = NewMemOp.getValue(1); |
0 |
11161 |
SDValue NewMemOpChain = NewMemOp.getValue(1); |
0 |
| 11162 |
return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain); |
0 |
11162 |
return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain); |
0 |
| 11163 |
} |
--- |
11163 |
} |
--- |
| 11164 |
|
--- |
11164 |
|
--- |
| 11165 |
SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op, |
0 |
11165 |
SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op, |
0 |
| 11166 |
Function **OutFunction) { |
--- |
11166 |
Function **OutFunction) { |
--- |
| 11167 |
assert(isa(Op) && "Node should be an ExternalSymbol"); |
0 |
11167 |
assert(isa(Op) && "Node should be an ExternalSymbol"); |
0 |
| 11168 |
|
--- |
11168 |
|
--- |
| 11169 |
auto *Symbol = cast(Op)->getSymbol(); |
0 |
11169 |
auto *Symbol = cast(Op)->getSymbol(); |
0 |
| 11170 |
auto *Module = MF->getFunction().getParent(); |
0 |
11170 |
auto *Module = MF->getFunction().getParent(); |
0 |
| 11171 |
auto *Function = Module->getFunction(Symbol); |
0 |
11171 |
auto *Function = Module->getFunction(Symbol); |
0 |
| 11172 |
|
--- |
11172 |
|
--- |
| 11173 |
if (OutFunction != nullptr) |
0 |
11173 |
if (OutFunction != nullptr) |
0 |
| 11174 |
*OutFunction = Function; |
0 |
11174 |
*OutFunction = Function; |
0 |
| 11175 |
|
--- |
11175 |
|
--- |
| 11176 |
if (Function != nullptr) { |
0 |
11176 |
if (Function != nullptr) { |
0 |
| 11177 |
auto PtrTy = TLI->getPointerTy(getDataLayout(), Function->getAddressSpace()); |
0 |
11177 |
auto PtrTy = TLI->getPointerTy(getDataLayout(), Function->getAddressSpace()); |
0 |
| 11178 |
return getGlobalAddress(Function, SDLoc(Op), PtrTy); |
0 |
11178 |
return getGlobalAddress(Function, SDLoc(Op), PtrTy); |
0 |
| 11179 |
} |
--- |
11179 |
} |
--- |
| 11180 |
|
--- |
11180 |
|
--- |
| 11181 |
std::string ErrorStr; |
0 |
11181 |
std::string ErrorStr; |
0 |
| 11182 |
raw_string_ostream ErrorFormatter(ErrorStr); |
0 |
11182 |
raw_string_ostream ErrorFormatter(ErrorStr); |
0 |
| 11183 |
ErrorFormatter << "Undefined external symbol "; |
0 |
11183 |
ErrorFormatter << "Undefined external symbol "; |
0 |
| 11184 |
ErrorFormatter << '"' << Symbol << '"'; |
0 |
11184 |
ErrorFormatter << '"' << Symbol << '"'; |
0 |
| 11185 |
report_fatal_error(Twine(ErrorFormatter.str())); |
0 |
11185 |
report_fatal_error(Twine(ErrorFormatter.str())); |
0 |
| 11186 |
} |
--- |
11186 |
} |
--- |
| 11187 |
|
--- |
11187 |
|
--- |
| 11188 |
//===----------------------------------------------------------------------===// |
--- |
11188 |
//===----------------------------------------------------------------------===// |
--- |
| 11189 |
// SDNode Class |
--- |
11189 |
// SDNode Class |
--- |
| 11190 |
//===----------------------------------------------------------------------===// |
--- |
11190 |
//===----------------------------------------------------------------------===// |
--- |
| 11191 |
|
--- |
11191 |
|
--- |
| 11192 |
bool llvm::isNullConstant(SDValue V) { |
2 |
11192 |
bool llvm::isNullConstant(SDValue V) { |
2 |
| 11193 |
ConstantSDNode *Const = dyn_cast(V); |
2 |
11193 |
ConstantSDNode *Const = dyn_cast(V); |
2 |
| 11194 |
return Const != nullptr && Const->isZero(); |
2 |
11194 |
return Const != nullptr && Const->isZero(); |
2 |
| 11195 |
} |
--- |
11195 |
} |
--- |
| 11196 |
|
--- |
11196 |
|
--- |
| 11197 |
bool llvm::isNullFPConstant(SDValue V) { |
0 |
11197 |
bool llvm::isNullFPConstant(SDValue V) { |
0 |
| 11198 |
ConstantFPSDNode *Const = dyn_cast(V); |
0 |
11198 |
ConstantFPSDNode *Const = dyn_cast(V); |
0 |
| 11199 |
return Const != nullptr && Const->isZero() && !Const->isNegative(); |
0 |
11199 |
return Const != nullptr && Const->isZero() && !Const->isNegative(); |
0 |
| 11200 |
} |
--- |
11200 |
} |
--- |
| 11201 |
|
--- |
11201 |
|
--- |
| 11202 |
bool llvm::isAllOnesConstant(SDValue V) { |
1 |
11202 |
bool llvm::isAllOnesConstant(SDValue V) { |
1 |
| 11203 |
ConstantSDNode *Const = dyn_cast(V); |
1 |
11203 |
ConstantSDNode *Const = dyn_cast(V); |
1 |
| 11204 |
return Const != nullptr && Const->isAllOnes(); |
1 |
11204 |
return Const != nullptr && Const->isAllOnes(); |
1 |
| 11205 |
} |
--- |
11205 |
} |
--- |
| 11206 |
|
--- |
11206 |
|
--- |
| 11207 |
bool llvm::isOneConstant(SDValue V) { |
0 |
11207 |
bool llvm::isOneConstant(SDValue V) { |
0 |
| 11208 |
ConstantSDNode *Const = dyn_cast(V); |
0 |
11208 |
ConstantSDNode *Const = dyn_cast(V); |
0 |
| 11209 |
return Const != nullptr && Const->isOne(); |
0 |
11209 |
return Const != nullptr && Const->isOne(); |
0 |
| 11210 |
} |
--- |
11210 |
} |
--- |
| 11211 |
|
--- |
11211 |
|
--- |
| 11212 |
bool llvm::isMinSignedConstant(SDValue V) { |
1 |
11212 |
bool llvm::isMinSignedConstant(SDValue V) { |
1 |
| 11213 |
ConstantSDNode *Const = dyn_cast(V); |
1 |
11213 |
ConstantSDNode *Const = dyn_cast(V); |
1 |
| 11214 |
return Const != nullptr && Const->isMinSignedValue(); |
1 |
11214 |
return Const != nullptr && Const->isMinSignedValue(); |
1 |
| 11215 |
} |
--- |
11215 |
} |
--- |
| 11216 |
|
--- |
11216 |
|
--- |
| 11217 |
bool llvm::isNeutralConstant(unsigned Opcode, SDNodeFlags Flags, SDValue V, |
0 |
11217 |
bool llvm::isNeutralConstant(unsigned Opcode, SDNodeFlags Flags, SDValue V, |
0 |
| 11218 |
unsigned OperandNo) { |
--- |
11218 |
unsigned OperandNo) { |
--- |
| 11219 |
// NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity(). |
--- |
11219 |
// NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity(). |
--- |
| 11220 |
// TODO: Target-specific opcodes could be added. |
--- |
11220 |
// TODO: Target-specific opcodes could be added. |
--- |
| 11221 |
if (auto *Const = isConstOrConstSplat(V)) { |
0 |
11221 |
if (auto *Const = isConstOrConstSplat(V)) { |
0 |
| 11222 |
switch (Opcode) { |
0 |
11222 |
switch (Opcode) { |
0 |
| 11223 |
case ISD::ADD: |
0 |
11223 |
case ISD::ADD: |
0 |
| 11224 |
case ISD::OR: |
--- |
11224 |
case ISD::OR: |
--- |
| 11225 |
case ISD::XOR: |
--- |
11225 |
case ISD::XOR: |
--- |
| 11226 |
case ISD::UMAX: |
--- |
11226 |
case ISD::UMAX: |
--- |
| 11227 |
return Const->isZero(); |
0 |
11227 |
return Const->isZero(); |
0 |
| 11228 |
case ISD::MUL: |
0 |
11228 |
case ISD::MUL: |
0 |
| 11229 |
return Const->isOne(); |
0 |
11229 |
return Const->isOne(); |
0 |
| 11230 |
case ISD::AND: |
0 |
11230 |
case ISD::AND: |
0 |
| 11231 |
case ISD::UMIN: |
--- |
11231 |
case ISD::UMIN: |
--- |
| 11232 |
return Const->isAllOnes(); |
0 |
11232 |
return Const->isAllOnes(); |
0 |
| 11233 |
case ISD::SMAX: |
0 |
11233 |
case ISD::SMAX: |
0 |
| 11234 |
return Const->isMinSignedValue(); |
0 |
11234 |
return Const->isMinSignedValue(); |
0 |
| 11235 |
case ISD::SMIN: |
0 |
11235 |
case ISD::SMIN: |
0 |
| 11236 |
return Const->isMaxSignedValue(); |
0 |
11236 |
return Const->isMaxSignedValue(); |
0 |
| 11237 |
case ISD::SUB: |
0 |
11237 |
case ISD::SUB: |
0 |
| 11238 |
case ISD::SHL: |
--- |
11238 |
case ISD::SHL: |
--- |
| 11239 |
case ISD::SRA: |
--- |
11239 |
case ISD::SRA: |
--- |
| 11240 |
case ISD::SRL: |
--- |
11240 |
case ISD::SRL: |
--- |
| 11241 |
return OperandNo == 1 && Const->isZero(); |
0 |
11241 |
return OperandNo == 1 && Const->isZero(); |
0 |
| 11242 |
case ISD::UDIV: |
0 |
11242 |
case ISD::UDIV: |
0 |
| 11243 |
case ISD::SDIV: |
--- |
11243 |
case ISD::SDIV: |
--- |
| 11244 |
return OperandNo == 1 && Const->isOne(); |
0 |
11244 |
return OperandNo == 1 && Const->isOne(); |
0 |
| 11245 |
} |
--- |
11245 |
} |
--- |
| 11246 |
} else if (auto *ConstFP = isConstOrConstSplatFP(V)) { |
0 |
11246 |
} else if (auto *ConstFP = isConstOrConstSplatFP(V)) { |
0 |
| 11247 |
switch (Opcode) { |
0 |
11247 |
switch (Opcode) { |
0 |
| 11248 |
case ISD::FADD: |
0 |
11248 |
case ISD::FADD: |
0 |
| 11249 |
return ConstFP->isZero() && |
0 |
11249 |
return ConstFP->isZero() && |
0 |
| 11250 |
(Flags.hasNoSignedZeros() || ConstFP->isNegative()); |
0 |
11250 |
(Flags.hasNoSignedZeros() || ConstFP->isNegative()); |
0 |
| 11251 |
case ISD::FSUB: |
0 |
11251 |
case ISD::FSUB: |
0 |
| 11252 |
return OperandNo == 1 && ConstFP->isZero() && |
0 |
11252 |
return OperandNo == 1 && ConstFP->isZero() && |
0 |
| 11253 |
(Flags.hasNoSignedZeros() || !ConstFP->isNegative()); |
0 |
11253 |
(Flags.hasNoSignedZeros() || !ConstFP->isNegative()); |
0 |
| 11254 |
case ISD::FMUL: |
0 |
11254 |
case ISD::FMUL: |
0 |
| 11255 |
return ConstFP->isExactlyValue(1.0); |
0 |
11255 |
return ConstFP->isExactlyValue(1.0); |
0 |
| 11256 |
case ISD::FDIV: |
0 |
11256 |
case ISD::FDIV: |
0 |
| 11257 |
return OperandNo == 1 && ConstFP->isExactlyValue(1.0); |
0 |
11257 |
return OperandNo == 1 && ConstFP->isExactlyValue(1.0); |
0 |
| 11258 |
case ISD::FMINNUM: |
0 |
11258 |
case ISD::FMINNUM: |
0 |
| 11259 |
case ISD::FMAXNUM: { |
--- |
11259 |
case ISD::FMAXNUM: { |
--- |
| 11260 |
// Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF. |
--- |
11260 |
// Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF. |
--- |
| 11261 |
EVT VT = V.getValueType(); |
0 |
11261 |
EVT VT = V.getValueType(); |
0 |
| 11262 |
const fltSemantics &Semantics = SelectionDAG::EVTToAPFloatSemantics(VT); |
0 |
11262 |
const fltSemantics &Semantics = SelectionDAG::EVTToAPFloatSemantics(VT); |
0 |
| 11263 |
APFloat NeutralAF = !Flags.hasNoNaNs() |
0 |
11263 |
APFloat NeutralAF = !Flags.hasNoNaNs() |
0 |
| 11264 |
? APFloat::getQNaN(Semantics) |
--- |
11264 |
? APFloat::getQNaN(Semantics) |
--- |
| 11265 |
: !Flags.hasNoInfs() |
0 |
11265 |
: !Flags.hasNoInfs() |
0 |
| 11266 |
? APFloat::getInf(Semantics) |
--- |
11266 |
? APFloat::getInf(Semantics) |
--- |
| 11267 |
: APFloat::getLargest(Semantics); |
0 |
11267 |
: APFloat::getLargest(Semantics); |
0 |
| 11268 |
if (Opcode == ISD::FMAXNUM) |
0 |
11268 |
if (Opcode == ISD::FMAXNUM) |
0 |
| 11269 |
NeutralAF.changeSign(); |
0 |
11269 |
NeutralAF.changeSign(); |
0 |
| 11270 |
|
--- |
11270 |
|
--- |
| 11271 |
return ConstFP->isExactlyValue(NeutralAF); |
0 |
11271 |
return ConstFP->isExactlyValue(NeutralAF); |
0 |
| 11272 |
} |
0 |
11272 |
} |
0 |
| 11273 |
} |
--- |
11273 |
} |
--- |
| 11274 |
} |
--- |
11274 |
} |
--- |
| 11275 |
return false; |
0 |
11275 |
return false; |
0 |
| 11276 |
} |
--- |
11276 |
} |
--- |
| 11277 |
|
--- |
11277 |
|
--- |
| 11278 |
SDValue llvm::peekThroughBitcasts(SDValue V) { |
0 |
11278 |
SDValue llvm::peekThroughBitcasts(SDValue V) { |
0 |
| 11279 |
while (V.getOpcode() == ISD::BITCAST) |
0 |
11279 |
while (V.getOpcode() == ISD::BITCAST) |
0 |
| 11280 |
V = V.getOperand(0); |
0 |
11280 |
V = V.getOperand(0); |
0 |
| 11281 |
return V; |
0 |
11281 |
return V; |
0 |
| 11282 |
} |
--- |
11282 |
} |
--- |
| 11283 |
|
--- |
11283 |
|
--- |
| 11284 |
SDValue llvm::peekThroughOneUseBitcasts(SDValue V) { |
0 |
11284 |
SDValue llvm::peekThroughOneUseBitcasts(SDValue V) { |
0 |
| 11285 |
while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse()) |
0 |
11285 |
while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse()) |
0 |
| 11286 |
V = V.getOperand(0); |
0 |
11286 |
V = V.getOperand(0); |
0 |
| 11287 |
return V; |
0 |
11287 |
return V; |
0 |
| 11288 |
} |
--- |
11288 |
} |
--- |
| 11289 |
|
--- |
11289 |
|
--- |
| 11290 |
SDValue llvm::peekThroughExtractSubvectors(SDValue V) { |
0 |
11290 |
SDValue llvm::peekThroughExtractSubvectors(SDValue V) { |
0 |
| 11291 |
while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR) |
0 |
11291 |
while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR) |
0 |
| 11292 |
V = V.getOperand(0); |
0 |
11292 |
V = V.getOperand(0); |
0 |
| 11293 |
return V; |
0 |
11293 |
return V; |
0 |
| 11294 |
} |
--- |
11294 |
} |
--- |
| 11295 |
|
--- |
11295 |
|
--- |
| 11296 |
SDValue llvm::peekThroughTruncates(SDValue V) { |
0 |
11296 |
SDValue llvm::peekThroughTruncates(SDValue V) { |
0 |
| 11297 |
while (V.getOpcode() == ISD::TRUNCATE) |
0 |
11297 |
while (V.getOpcode() == ISD::TRUNCATE) |
0 |
| 11298 |
V = V.getOperand(0); |
0 |
11298 |
V = V.getOperand(0); |
0 |
| 11299 |
return V; |
0 |
11299 |
return V; |
0 |
| 11300 |
} |
--- |
11300 |
} |
--- |
| 11301 |
|
--- |
11301 |
|
--- |
| 11302 |
bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) { |
8 |
11302 |
bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) { |
8 |
| 11303 |
if (V.getOpcode() != ISD::XOR) |
8 |
11303 |
if (V.getOpcode() != ISD::XOR) |
8 |
| 11304 |
return false; |
8 |
11304 |
return false; |
8 |
| 11305 |
V = peekThroughBitcasts(V.getOperand(1)); |
0 |
11305 |
V = peekThroughBitcasts(V.getOperand(1)); |
0 |
| 11306 |
unsigned NumBits = V.getScalarValueSizeInBits(); |
0 |
11306 |
unsigned NumBits = V.getScalarValueSizeInBits(); |
0 |
| 11307 |
ConstantSDNode *C = |
--- |
11307 |
ConstantSDNode *C = |
--- |
| 11308 |
isConstOrConstSplat(V, AllowUndefs, /*AllowTruncation*/ true); |
0 |
11308 |
isConstOrConstSplat(V, AllowUndefs, /*AllowTruncation*/ true); |
0 |
| 11309 |
return C && (C->getAPIntValue().countr_one() >= NumBits); |
0 |
11309 |
return C && (C->getAPIntValue().countr_one() >= NumBits); |
0 |
| 11310 |
} |
--- |
11310 |
} |
--- |
| 11311 |
|
--- |
11311 |
|
--- |
| 11312 |
ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs, |
30 |
11312 |
ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs, |
30 |
| 11313 |
bool AllowTruncation) { |
--- |
11313 |
bool AllowTruncation) { |
--- |
| 11314 |
EVT VT = N.getValueType(); |
30 |
11314 |
EVT VT = N.getValueType(); |
30 |
| 11315 |
APInt DemandedElts = VT.isFixedLengthVector() |
30 |
11315 |
APInt DemandedElts = VT.isFixedLengthVector() |
30 |
| 11316 |
? APInt::getAllOnes(VT.getVectorMinNumElements()) |
--- |
11316 |
? APInt::getAllOnes(VT.getVectorMinNumElements()) |
--- |
| 11317 |
: APInt(1, 1); |
30 |
11317 |
: APInt(1, 1); |
30 |
| 11318 |
return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation); |
30 |
11318 |
return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation); |
30 |
| 11319 |
} |
30 |
11319 |
} |
30 |
| 11320 |
|
--- |
11320 |
|
--- |
| 11321 |
ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts, |
31 |
11321 |
ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts, |
31 |
| 11322 |
bool AllowUndefs, |
--- |
11322 |
bool AllowUndefs, |
--- |
| 11323 |
bool AllowTruncation) { |
--- |
11323 |
bool AllowTruncation) { |
--- |
| 11324 |
if (ConstantSDNode *CN = dyn_cast(N)) |
31 |
11324 |
if (ConstantSDNode *CN = dyn_cast(N)) |
31 |
| 11325 |
return CN; |
16 |
11325 |
return CN; |
16 |
| 11326 |
|
--- |
11326 |
|
--- |
| 11327 |
// SplatVectors can truncate their operands. Ignore that case here unless |
--- |
11327 |
// SplatVectors can truncate their operands. Ignore that case here unless |
--- |
| 11328 |
// AllowTruncation is set. |
--- |
11328 |
// AllowTruncation is set. |
--- |
| 11329 |
if (N->getOpcode() == ISD::SPLAT_VECTOR) { |
15 |
11329 |
if (N->getOpcode() == ISD::SPLAT_VECTOR) { |
15 |
| 11330 |
EVT VecEltVT = N->getValueType(0).getVectorElementType(); |
0 |
11330 |
EVT VecEltVT = N->getValueType(0).getVectorElementType(); |
0 |
| 11331 |
if (auto *CN = dyn_cast(N->getOperand(0))) { |
0 |
11331 |
if (auto *CN = dyn_cast(N->getOperand(0))) { |
0 |
| 11332 |
EVT CVT = CN->getValueType(0); |
0 |
11332 |
EVT CVT = CN->getValueType(0); |
0 |
| 11333 |
assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension"); |
0 |
11333 |
assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension"); |
0 |
| 11334 |
if (AllowTruncation || CVT == VecEltVT) |
0 |
11334 |
if (AllowTruncation || CVT == VecEltVT) |
0 |
| 11335 |
return CN; |
0 |
11335 |
return CN; |
0 |
| 11336 |
} |
--- |
11336 |
} |
--- |
| 11337 |
} |
--- |
11337 |
} |
--- |
| 11338 |
|
--- |
11338 |
|
--- |
| 11339 |
if (BuildVectorSDNode *BV = dyn_cast(N)) { |
15 |
11339 |
if (BuildVectorSDNode *BV = dyn_cast(N)) { |
15 |
| 11340 |
BitVector UndefElements; |
0 |
11340 |
BitVector UndefElements; |
0 |
| 11341 |
ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements); |
0 |
11341 |
ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements); |
0 |
| 11342 |
|
--- |
11342 |
|
--- |
| 11343 |
// BuildVectors can truncate their operands. Ignore that case here unless |
--- |
11343 |
// BuildVectors can truncate their operands. Ignore that case here unless |
--- |
| 11344 |
// AllowTruncation is set. |
--- |
11344 |
// AllowTruncation is set. |
--- |
| 11345 |
// TODO: Look into whether we should allow UndefElements in non-DemandedElts |
--- |
11345 |
// TODO: Look into whether we should allow UndefElements in non-DemandedElts |
--- |
| 11346 |
if (CN && (UndefElements.none() || AllowUndefs)) { |
0 |
11346 |
if (CN && (UndefElements.none() || AllowUndefs)) { |
0 |
| 11347 |
EVT CVT = CN->getValueType(0); |
0 |
11347 |
EVT CVT = CN->getValueType(0); |
0 |
| 11348 |
EVT NSVT = N.getValueType().getScalarType(); |
0 |
11348 |
EVT NSVT = N.getValueType().getScalarType(); |
0 |
| 11349 |
assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension"); |
0 |
11349 |
assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension"); |
0 |
| 11350 |
if (AllowTruncation || (CVT == NSVT)) |
0 |
11350 |
if (AllowTruncation || (CVT == NSVT)) |
0 |
| 11351 |
return CN; |
0 |
11351 |
return CN; |
0 |
| 11352 |
} |
--- |
11352 |
} |
--- |
| 11353 |
} |
0 |
11353 |
} |
0 |
| 11354 |
|
--- |
11354 |
|
--- |
| 11355 |
return nullptr; |
15 |
11355 |
return nullptr; |
15 |
| 11356 |
} |
--- |
11356 |
} |
--- |
| 11357 |
|
--- |
11357 |
|
--- |
| 11358 |
ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) { |
16 |
11358 |
ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) { |
16 |
| 11359 |
EVT VT = N.getValueType(); |
16 |
11359 |
EVT VT = N.getValueType(); |
16 |
| 11360 |
APInt DemandedElts = VT.isFixedLengthVector() |
16 |
11360 |
APInt DemandedElts = VT.isFixedLengthVector() |
16 |
| 11361 |
? APInt::getAllOnes(VT.getVectorMinNumElements()) |
--- |
11361 |
? APInt::getAllOnes(VT.getVectorMinNumElements()) |
--- |
| 11362 |
: APInt(1, 1); |
16 |
11362 |
: APInt(1, 1); |
16 |
| 11363 |
return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs); |
16 |
11363 |
return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs); |
16 |
| 11364 |
} |
16 |
11364 |
} |
16 |
| 11365 |
|
--- |
11365 |
|
--- |
| 11366 |
ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, |
16 |
11366 |
ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, |
16 |
| 11367 |
const APInt &DemandedElts, |
--- |
11367 |
const APInt &DemandedElts, |
--- |
| 11368 |
bool AllowUndefs) { |
--- |
11368 |
bool AllowUndefs) { |
--- |
| 11369 |
if (ConstantFPSDNode *CN = dyn_cast(N)) |
16 |
11369 |
if (ConstantFPSDNode *CN = dyn_cast(N)) |
16 |
| 11370 |
return CN; |
0 |
11370 |
return CN; |
0 |
| 11371 |
|
--- |
11371 |
|
--- |
| 11372 |
if (BuildVectorSDNode *BV = dyn_cast(N)) { |
16 |
11372 |
if (BuildVectorSDNode *BV = dyn_cast(N)) { |
16 |
| 11373 |
BitVector UndefElements; |
0 |
11373 |
BitVector UndefElements; |
0 |
| 11374 |
ConstantFPSDNode *CN = |
--- |
11374 |
ConstantFPSDNode *CN = |
--- |
| 11375 |
BV->getConstantFPSplatNode(DemandedElts, &UndefElements); |
0 |
11375 |
BV->getConstantFPSplatNode(DemandedElts, &UndefElements); |
0 |
| 11376 |
// TODO: Look into whether we should allow UndefElements in non-DemandedElts |
--- |
11376 |
// TODO: Look into whether we should allow UndefElements in non-DemandedElts |
--- |
| 11377 |
if (CN && (UndefElements.none() || AllowUndefs)) |
0 |
11377 |
if (CN && (UndefElements.none() || AllowUndefs)) |
0 |
| 11378 |
return CN; |
0 |
11378 |
return CN; |
0 |
| 11379 |
} |
0 |
11379 |
} |
0 |
| 11380 |
|
--- |
11380 |
|
--- |
| 11381 |
if (N.getOpcode() == ISD::SPLAT_VECTOR) |
16 |
11381 |
if (N.getOpcode() == ISD::SPLAT_VECTOR) |
16 |
| 11382 |
if (ConstantFPSDNode *CN = dyn_cast(N.getOperand(0))) |
0 |
11382 |
if (ConstantFPSDNode *CN = dyn_cast(N.getOperand(0))) |
0 |
| 11383 |
return CN; |
0 |
11383 |
return CN; |
0 |
| 11384 |
|
--- |
11384 |
|
--- |
| 11385 |
return nullptr; |
16 |
11385 |
return nullptr; |
16 |
| 11386 |
} |
--- |
11386 |
} |
--- |
| 11387 |
|
--- |
11387 |
|
--- |
| 11388 |
bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) { |
0 |
11388 |
bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) { |
0 |
| 11389 |
// TODO: may want to use peekThroughBitcast() here. |
--- |
11389 |
// TODO: may want to use peekThroughBitcast() here. |
--- |
| 11390 |
ConstantSDNode *C = |
--- |
11390 |
ConstantSDNode *C = |
--- |
| 11391 |
isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true); |
0 |
11391 |
isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true); |
0 |
| 11392 |
return C && C->isZero(); |
0 |
11392 |
return C && C->isZero(); |
0 |
| 11393 |
} |
--- |
11393 |
} |
--- |
| 11394 |
|
--- |
11394 |
|
--- |
| 11395 |
bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) { |
1 |
11395 |
bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) { |
1 |
| 11396 |
ConstantSDNode *C = |
--- |
11396 |
ConstantSDNode *C = |
--- |
| 11397 |
isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true); |
1 |
11397 |
isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true); |
1 |
| 11398 |
return C && C->isOne(); |
1 |
11398 |
return C && C->isOne(); |
1 |
| 11399 |
} |
--- |
11399 |
} |
--- |
| 11400 |
|
--- |
11400 |
|
--- |
| 11401 |
bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) { |
0 |
11401 |
bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) { |
0 |
| 11402 |
N = peekThroughBitcasts(N); |
0 |
11402 |
N = peekThroughBitcasts(N); |
0 |
| 11403 |
unsigned BitWidth = N.getScalarValueSizeInBits(); |
0 |
11403 |
unsigned BitWidth = N.getScalarValueSizeInBits(); |
0 |
| 11404 |
ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs); |
0 |
11404 |
ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs); |
0 |
| 11405 |
return C && C->isAllOnes() && C->getValueSizeInBits(0) == BitWidth; |
0 |
11405 |
return C && C->isAllOnes() && C->getValueSizeInBits(0) == BitWidth; |
0 |
| 11406 |
} |
--- |
11406 |
} |
--- |
| 11407 |
|
--- |
11407 |
|
--- |
| 11408 |
HandleSDNode::~HandleSDNode() { |
39 |
11408 |
HandleSDNode::~HandleSDNode() { |
39 |
| 11409 |
DropOperands(); |
39 |
11409 |
DropOperands(); |
39 |
| 11410 |
} |
39 |
11410 |
} |
39 |
| 11411 |
|
--- |
11411 |
|
--- |
| 11412 |
GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order, |
0 |
11412 |
GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order, |
0 |
| 11413 |
const DebugLoc &DL, |
--- |
11413 |
const DebugLoc &DL, |
--- |
| 11414 |
const GlobalValue *GA, EVT VT, |
--- |
11414 |
const GlobalValue *GA, EVT VT, |
--- |
| 11415 |
int64_t o, unsigned TF) |
--- |
11415 |
int64_t o, unsigned TF) |
--- |
| 11416 |
: SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { |
0 |
11416 |
: SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { |
0 |
| 11417 |
TheGlobal = GA; |
0 |
11417 |
TheGlobal = GA; |
0 |
| 11418 |
} |
0 |
11418 |
} |
0 |
| 11419 |
|
--- |
11419 |
|
--- |
| 11420 |
AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, |
0 |
11420 |
AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, |
0 |
| 11421 |
EVT VT, unsigned SrcAS, |
--- |
11421 |
EVT VT, unsigned SrcAS, |
--- |
| 11422 |
unsigned DestAS) |
--- |
11422 |
unsigned DestAS) |
--- |
| 11423 |
: SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)), |
--- |
11423 |
: SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)), |
--- |
| 11424 |
SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {} |
0 |
11424 |
SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {} |
0 |
| 11425 |
|
--- |
11425 |
|
--- |
| 11426 |
MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, |
18 |
11426 |
MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, |
18 |
| 11427 |
SDVTList VTs, EVT memvt, MachineMemOperand *mmo) |
--- |
11427 |
SDVTList VTs, EVT memvt, MachineMemOperand *mmo) |
--- |
| 11428 |
: SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { |
18 |
11428 |
: SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { |
18 |
| 11429 |
MemSDNodeBits.IsVolatile = MMO->isVolatile(); |
18 |
11429 |
MemSDNodeBits.IsVolatile = MMO->isVolatile(); |
18 |
| 11430 |
MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal(); |
18 |
11430 |
MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal(); |
18 |
| 11431 |
MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable(); |
18 |
11431 |
MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable(); |
18 |
| 11432 |
MemSDNodeBits.IsInvariant = MMO->isInvariant(); |
18 |
11432 |
MemSDNodeBits.IsInvariant = MMO->isInvariant(); |
18 |
| 11433 |
|
--- |
11433 |
|
--- |
| 11434 |
// We check here that the size of the memory operand fits within the size of |
--- |
11434 |
// We check here that the size of the memory operand fits within the size of |
--- |
| 11435 |
// the MMO. This is because the MMO might indicate only a possible address |
--- |
11435 |
// the MMO. This is because the MMO might indicate only a possible address |
--- |
| 11436 |
// range instead of specifying the affected memory addresses precisely. |
--- |
11436 |
// range instead of specifying the affected memory addresses precisely. |
--- |
| 11437 |
// TODO: Make MachineMemOperands aware of scalable vectors. |
--- |
11437 |
// TODO: Make MachineMemOperands aware of scalable vectors. |
--- |
| 11438 |
assert(memvt.getStoreSize().getKnownMinValue() <= MMO->getSize() && |
18 |
11438 |
assert(memvt.getStoreSize().getKnownMinValue() <= MMO->getSize() && |
18 |
| 11439 |
"Size mismatch!"); |
--- |
11439 |
"Size mismatch!"); |
--- |
| 11440 |
} |
18 |
11440 |
} |
18 |
| 11441 |
|
--- |
11441 |
|
--- |
| 11442 |
/// Profile - Gather unique data for the node. |
--- |
11442 |
/// Profile - Gather unique data for the node. |
--- |
| 11443 |
/// |
--- |
11443 |
/// |
--- |
| 11444 |
void SDNode::Profile(FoldingSetNodeID &ID) const { |
36 |
11444 |
void SDNode::Profile(FoldingSetNodeID &ID) const { |
49 |
| 11445 |
AddNodeIDNode(ID, this); |
36 |
11445 |
AddNodeIDNode(ID, this); |
49 |
| 11446 |
} |
36 |
11446 |
} |
49 |
| 11447 |
|
--- |
11447 |
|
--- |
| 11448 |
namespace { |
--- |
11448 |
namespace { |
--- |
| 11449 |
|
--- |
11449 |
|
--- |
| 11450 |
struct EVTArray { |
--- |
11450 |
struct EVTArray { |
--- |
| 11451 |
std::vector VTs; |
--- |
11451 |
std::vector VTs; |
--- |
| 11452 |
|
--- |
11452 |
|
--- |
| 11453 |
EVTArray() { |
1 |
11453 |
EVTArray() { |
1 |
| 11454 |
VTs.reserve(MVT::VALUETYPE_SIZE); |
1 |
11454 |
VTs.reserve(MVT::VALUETYPE_SIZE); |
1 |
| 11455 |
for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i) |
199 |
11455 |
for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i) |
199 |
| 11456 |
VTs.push_back(MVT((MVT::SimpleValueType)i)); |
198 |
11456 |
VTs.push_back(MVT((MVT::SimpleValueType)i)); |
198 |
| 11457 |
} |
1 |
11457 |
} |
1 |
| 11458 |
}; |
--- |
11458 |
}; |
--- |
| 11459 |
|
--- |
11459 |
|
--- |
| 11460 |
} // end anonymous namespace |
--- |
11460 |
} // end anonymous namespace |
--- |
| 11461 |
|
--- |
11461 |
|
--- |
| 11462 |
/// getValueTypeList - Return a pointer to the specified value type. |
--- |
11462 |
/// getValueTypeList - Return a pointer to the specified value type. |
--- |
| 11463 |
/// |
--- |
11463 |
/// |
--- |
| 11464 |
const EVT *SDNode::getValueTypeList(EVT VT) { |
160 |
11464 |
const EVT *SDNode::getValueTypeList(EVT VT) { |
160 |
| 11465 |
static std::set EVTs; |
160 |
11465 |
static std::set EVTs; |
160 |
| 11466 |
static EVTArray SimpleVTArray; |
160 |
11466 |
static EVTArray SimpleVTArray; |
160 |
| 11467 |
static sys::SmartMutex VTMutex; |
--- |
11467 |
static sys::SmartMutex VTMutex; |
--- |
| 11468 |
|
--- |
11468 |
|
--- |
| 11469 |
if (VT.isExtended()) { |
160 |
11469 |
if (VT.isExtended()) { |
160 |
| 11470 |
sys::SmartScopedLock Lock(VTMutex); |
0 |
11470 |
sys::SmartScopedLock Lock(VTMutex); |
0 |
| 11471 |
return &(*EVTs.insert(VT).first); |
0 |
11471 |
return &(*EVTs.insert(VT).first); |
0 |
| 11472 |
} |
0 |
11472 |
} |
0 |
| 11473 |
assert(VT.getSimpleVT() < MVT::VALUETYPE_SIZE && "Value type out of range!"); |
160 |
11473 |
assert(VT.getSimpleVT() < MVT::VALUETYPE_SIZE && "Value type out of range!"); |
160 |
| 11474 |
return &SimpleVTArray.VTs[VT.getSimpleVT().SimpleTy]; |
160 |
11474 |
return &SimpleVTArray.VTs[VT.getSimpleVT().SimpleTy]; |
160 |
| 11475 |
} |
--- |
11475 |
} |
--- |
| 11476 |
|
--- |
11476 |
|
--- |
| 11477 |
/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the |
--- |
11477 |
/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the |
--- |
| 11478 |
/// indicated value. This method ignores uses of other values defined by this |
--- |
11478 |
/// indicated value. This method ignores uses of other values defined by this |
--- |
| 11479 |
/// operation. |
--- |
11479 |
/// operation. |
--- |
| 11480 |
bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { |
23 |
11480 |
bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { |
23 |
| 11481 |
assert(Value < getNumValues() && "Bad value!"); |
23 |
11481 |
assert(Value < getNumValues() && "Bad value!"); |
23 |
| 11482 |
|
--- |
11482 |
|
--- |
| 11483 |
// TODO: Only iterate over uses of a given value of the node |
--- |
11483 |
// TODO: Only iterate over uses of a given value of the node |
--- |
| 11484 |
for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { |
52 |
11484 |
for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { |
52 |
| 11485 |
if (UI.getUse().getResNo() == Value) { |
29 |
11485 |
if (UI.getUse().getResNo() == Value) { |
29 |
| 11486 |
if (NUses == 0) |
23 |
11486 |
if (NUses == 0) |
23 |
| 11487 |
return false; |
0 |
11487 |
return false; |
0 |
| 11488 |
--NUses; |
23 |
11488 |
--NUses; |
23 |
| 11489 |
} |
--- |
11489 |
} |
--- |
| 11490 |
} |
--- |
11490 |
} |
--- |
| 11491 |
|
--- |
11491 |
|
--- |
| 11492 |
// Found exactly the right number of uses? |
--- |
11492 |
// Found exactly the right number of uses? |
--- |
| 11493 |
return NUses == 0; |
23 |
11493 |
return NUses == 0; |
23 |
| 11494 |
} |
--- |
11494 |
} |
--- |
| 11495 |
|
--- |
11495 |
|
--- |
| 11496 |
/// hasAnyUseOfValue - Return true if there are any use of the indicated |
--- |
11496 |
/// hasAnyUseOfValue - Return true if there are any use of the indicated |
--- |
| 11497 |
/// value. This method ignores uses of other values defined by this operation. |
--- |
11497 |
/// value. This method ignores uses of other values defined by this operation. |
--- |
| 11498 |
bool SDNode::hasAnyUseOfValue(unsigned Value) const { |
23 |
11498 |
bool SDNode::hasAnyUseOfValue(unsigned Value) const { |
23 |
| 11499 |
assert(Value < getNumValues() && "Bad value!"); |
23 |
11499 |
assert(Value < getNumValues() && "Bad value!"); |
23 |
| 11500 |
|
--- |
11500 |
|
--- |
| 11501 |
for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) |
25 |
11501 |
for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) |
25 |
| 11502 |
if (UI.getUse().getResNo() == Value) |
24 |
11502 |
if (UI.getUse().getResNo() == Value) |
24 |
| 11503 |
return true; |
22 |
11503 |
return true; |
22 |
| 11504 |
|
--- |
11504 |
|
--- |
| 11505 |
return false; |
1 |
11505 |
return false; |
1 |
| 11506 |
} |
--- |
11506 |
} |
--- |
| 11507 |
|
--- |
11507 |
|
--- |
| 11508 |
/// isOnlyUserOf - Return true if this node is the only use of N. |
--- |
11508 |
/// isOnlyUserOf - Return true if this node is the only use of N. |
--- |
| 11509 |
bool SDNode::isOnlyUserOf(const SDNode *N) const { |
0 |
11509 |
bool SDNode::isOnlyUserOf(const SDNode *N) const { |
0 |
| 11510 |
bool Seen = false; |
0 |
11510 |
bool Seen = false; |
0 |
| 11511 |
for (const SDNode *User : N->uses()) { |
0 |
11511 |
for (const SDNode *User : N->uses()) { |
0 |
| 11512 |
if (User == this) |
0 |
11512 |
if (User == this) |
0 |
| 11513 |
Seen = true; |
0 |
11513 |
Seen = true; |
0 |
| 11514 |
else |
--- |
11514 |
else |
--- |
| 11515 |
return false; |
0 |
11515 |
return false; |
0 |
| 11516 |
} |
--- |
11516 |
} |
--- |
| 11517 |
|
--- |
11517 |
|
--- |
| 11518 |
return Seen; |
0 |
11518 |
return Seen; |
0 |
| 11519 |
} |
--- |
11519 |
} |
--- |
| 11520 |
|
--- |
11520 |
|
--- |
| 11521 |
/// Return true if the only users of N are contained in Nodes. |
--- |
11521 |
/// Return true if the only users of N are contained in Nodes. |
--- |
| 11522 |
bool SDNode::areOnlyUsersOf(ArrayRef Nodes, const SDNode *N) { |
0 |
11522 |
bool SDNode::areOnlyUsersOf(ArrayRef Nodes, const SDNode *N) { |
0 |
| 11523 |
bool Seen = false; |
0 |
11523 |
bool Seen = false; |
0 |
| 11524 |
for (const SDNode *User : N->uses()) { |
0 |
11524 |
for (const SDNode *User : N->uses()) { |
0 |
| 11525 |
if (llvm::is_contained(Nodes, User)) |
0 |
11525 |
if (llvm::is_contained(Nodes, User)) |
0 |
| 11526 |
Seen = true; |
0 |
11526 |
Seen = true; |
0 |
| 11527 |
else |
--- |
11527 |
else |
--- |
| 11528 |
return false; |
0 |
11528 |
return false; |
0 |
| 11529 |
} |
--- |
11529 |
} |
--- |
| 11530 |
|
--- |
11530 |
|
--- |
| 11531 |
return Seen; |
0 |
11531 |
return Seen; |
0 |
| 11532 |
} |
--- |
11532 |
} |
--- |
| 11533 |
|
--- |
11533 |
|
--- |
| 11534 |
/// isOperand - Return true if this node is an operand of N. |
--- |
11534 |
/// isOperand - Return true if this node is an operand of N. |
--- |
| 11535 |
bool SDValue::isOperandOf(const SDNode *N) const { |
0 |
11535 |
bool SDValue::isOperandOf(const SDNode *N) const { |
0 |
| 11536 |
return is_contained(N->op_values(), *this); |
0 |
11536 |
return is_contained(N->op_values(), *this); |
0 |
| 11537 |
} |
--- |
11537 |
} |
--- |
| 11538 |
|
--- |
11538 |
|
--- |
| 11539 |
bool SDNode::isOperandOf(const SDNode *N) const { |
0 |
11539 |
bool SDNode::isOperandOf(const SDNode *N) const { |
0 |
| 11540 |
return any_of(N->op_values(), |
0 |
11540 |
return any_of(N->op_values(), |
0 |
| 11541 |
[this](SDValue Op) { return this == Op.getNode(); }); |
0 |
11541 |
[this](SDValue Op) { return this == Op.getNode(); }); |
0 |
| 11542 |
} |
--- |
11542 |
} |
--- |
| 11543 |
|
--- |
11543 |
|
--- |
| 11544 |
/// reachesChainWithoutSideEffects - Return true if this operand (which must |
--- |
11544 |
/// reachesChainWithoutSideEffects - Return true if this operand (which must |
--- |
| 11545 |
/// be a chain) reaches the specified operand without crossing any |
--- |
11545 |
/// be a chain) reaches the specified operand without crossing any |
--- |
| 11546 |
/// side-effecting instructions on any chain path. In practice, this looks |
--- |
11546 |
/// side-effecting instructions on any chain path. In practice, this looks |
--- |
| 11547 |
/// through token factors and non-volatile loads. In order to remain efficient, |
--- |
11547 |
/// through token factors and non-volatile loads. In order to remain efficient, |
--- |
| 11548 |
/// this only looks a couple of nodes in, it does not do an exhaustive search. |
--- |
11548 |
/// this only looks a couple of nodes in, it does not do an exhaustive search. |
--- |
| 11549 |
/// |
--- |
11549 |
/// |
--- |
| 11550 |
/// Note that we only need to examine chains when we're searching for |
--- |
11550 |
/// Note that we only need to examine chains when we're searching for |
--- |
| 11551 |
/// side-effects; SelectionDAG requires that all side-effects are represented |
--- |
11551 |
/// side-effects; SelectionDAG requires that all side-effects are represented |
--- |
| 11552 |
/// by chains, even if another operand would force a specific ordering. This |
--- |
11552 |
/// by chains, even if another operand would force a specific ordering. This |
--- |
| 11553 |
/// constraint is necessary to allow transformations like splitting loads. |
--- |
11553 |
/// constraint is necessary to allow transformations like splitting loads. |
--- |
| 11554 |
bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, |
0 |
11554 |
bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, |
0 |
| 11555 |
unsigned Depth) const { |
--- |
11555 |
unsigned Depth) const { |
--- |
| 11556 |
if (*this == Dest) return true; |
0 |
11556 |
if (*this == Dest) return true; |
0 |
| 11557 |
|
--- |
11557 |
|
--- |
| 11558 |
// Don't search too deeply, we just want to be able to see through |
--- |
11558 |
// Don't search too deeply, we just want to be able to see through |
--- |
| 11559 |
// TokenFactor's etc. |
--- |
11559 |
// TokenFactor's etc. |
--- |
| 11560 |
if (Depth == 0) return false; |
0 |
11560 |
if (Depth == 0) return false; |
0 |
| 11561 |
|
--- |
11561 |
|
--- |
| 11562 |
// If this is a token factor, all inputs to the TF happen in parallel. |
--- |
11562 |
// If this is a token factor, all inputs to the TF happen in parallel. |
--- |
| 11563 |
if (getOpcode() == ISD::TokenFactor) { |
0 |
11563 |
if (getOpcode() == ISD::TokenFactor) { |
0 |
| 11564 |
// First, try a shallow search. |
--- |
11564 |
// First, try a shallow search. |
--- |
| 11565 |
if (is_contained((*this)->ops(), Dest)) { |
0 |
11565 |
if (is_contained((*this)->ops(), Dest)) { |
0 |
| 11566 |
// We found the chain we want as an operand of this TokenFactor. |
--- |
11566 |
// We found the chain we want as an operand of this TokenFactor. |
--- |
| 11567 |
// Essentially, we reach the chain without side-effects if we could |
--- |
11567 |
// Essentially, we reach the chain without side-effects if we could |
--- |
| 11568 |
// serialize the TokenFactor into a simple chain of operations with |
--- |
11568 |
// serialize the TokenFactor into a simple chain of operations with |
--- |
| 11569 |
// Dest as the last operation. This is automatically true if the |
--- |
11569 |
// Dest as the last operation. This is automatically true if the |
--- |
| 11570 |
// chain has one use: there are no other ordering constraints. |
--- |
11570 |
// chain has one use: there are no other ordering constraints. |
--- |
| 11571 |
// If the chain has more than one use, we give up: some other |
--- |
11571 |
// If the chain has more than one use, we give up: some other |
--- |
| 11572 |
// use of Dest might force a side-effect between Dest and the current |
--- |
11572 |
// use of Dest might force a side-effect between Dest and the current |
--- |
| 11573 |
// node. |
--- |
11573 |
// node. |
--- |
| 11574 |
if (Dest.hasOneUse()) |
0 |
11574 |
if (Dest.hasOneUse()) |
0 |
| 11575 |
return true; |
0 |
11575 |
return true; |
0 |
| 11576 |
} |
--- |
11576 |
} |
--- |
| 11577 |
// Next, try a deep search: check whether every operand of the TokenFactor |
--- |
11577 |
// Next, try a deep search: check whether every operand of the TokenFactor |
--- |
| 11578 |
// reaches Dest. |
--- |
11578 |
// reaches Dest. |
--- |
| 11579 |
return llvm::all_of((*this)->ops(), [=](SDValue Op) { |
0 |
11579 |
return llvm::all_of((*this)->ops(), [=](SDValue Op) { |
0 |
| 11580 |
return Op.reachesChainWithoutSideEffects(Dest, Depth - 1); |
0 |
11580 |
return Op.reachesChainWithoutSideEffects(Dest, Depth - 1); |
0 |
| 11581 |
}); |
0 |
11581 |
}); |
0 |
| 11582 |
} |
--- |
11582 |
} |
--- |
| 11583 |
|
--- |
11583 |
|
--- |
| 11584 |
// Loads don't have side effects, look through them. |
--- |
11584 |
// Loads don't have side effects, look through them. |
--- |
| 11585 |
if (LoadSDNode *Ld = dyn_cast(*this)) { |
0 |
11585 |
if (LoadSDNode *Ld = dyn_cast(*this)) { |
0 |
| 11586 |
if (Ld->isUnordered()) |
0 |
11586 |
if (Ld->isUnordered()) |
0 |
| 11587 |
return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1); |
0 |
11587 |
return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1); |
0 |
| 11588 |
} |
--- |
11588 |
} |
--- |
| 11589 |
return false; |
0 |
11589 |
return false; |
0 |
| 11590 |
} |
--- |
11590 |
} |
--- |
| 11591 |
|
--- |
11591 |
|
--- |
| 11592 |
bool SDNode::hasPredecessor(const SDNode *N) const { |
0 |
11592 |
bool SDNode::hasPredecessor(const SDNode *N) const { |
0 |
| 11593 |
SmallPtrSet Visited; |
0 |
11593 |
SmallPtrSet Visited; |
0 |
| 11594 |
SmallVector Worklist; |
0 |
11594 |
SmallVector Worklist; |
0 |
| 11595 |
Worklist.push_back(this); |
0 |
11595 |
Worklist.push_back(this); |
0 |
| 11596 |
return hasPredecessorHelper(N, Visited, Worklist); |
0 |
11596 |
return hasPredecessorHelper(N, Visited, Worklist); |
0 |
| 11597 |
} |
0 |
11597 |
} |
0 |
| 11598 |
|
--- |
11598 |
|
--- |
| 11599 |
void SDNode::intersectFlagsWith(const SDNodeFlags Flags) { |
0 |
11599 |
void SDNode::intersectFlagsWith(const SDNodeFlags Flags) { |
0 |
| 11600 |
this->Flags.intersectWith(Flags); |
0 |
11600 |
this->Flags.intersectWith(Flags); |
0 |
| 11601 |
} |
0 |
11601 |
} |
0 |
| 11602 |
|
--- |
11602 |
|
--- |
| 11603 |
SDValue |
--- |
11603 |
SDValue |
--- |
| 11604 |
SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, |
0 |
11604 |
SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, |
0 |
| 11605 |
ArrayRef CandidateBinOps, |
--- |
11605 |
ArrayRef CandidateBinOps, |
--- |
| 11606 |
bool AllowPartials) { |
--- |
11606 |
bool AllowPartials) { |
--- |
| 11607 |
// The pattern must end in an extract from index 0. |
--- |
11607 |
// The pattern must end in an extract from index 0. |
--- |
| 11608 |
if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT || |
0 |
11608 |
if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT || |
0 |
| 11609 |
!isNullConstant(Extract->getOperand(1))) |
0 |
11609 |
!isNullConstant(Extract->getOperand(1))) |
0 |
| 11610 |
return SDValue(); |
0 |
11610 |
return SDValue(); |
0 |
| 11611 |
|
--- |
11611 |
|
--- |
| 11612 |
// Match against one of the candidate binary ops. |
--- |
11612 |
// Match against one of the candidate binary ops. |
--- |
| 11613 |
SDValue Op = Extract->getOperand(0); |
0 |
11613 |
SDValue Op = Extract->getOperand(0); |
0 |
| 11614 |
if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) { |
0 |
11614 |
if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) { |
0 |
| 11615 |
return Op.getOpcode() == unsigned(BinOp); |
0 |
11615 |
return Op.getOpcode() == unsigned(BinOp); |
0 |
| 11616 |
})) |
--- |
11616 |
})) |
--- |
| 11617 |
return SDValue(); |
0 |
11617 |
return SDValue(); |
0 |
| 11618 |
|
--- |
11618 |
|
--- |
| 11619 |
// Floating-point reductions may require relaxed constraints on the final step |
--- |
11619 |
// Floating-point reductions may require relaxed constraints on the final step |
--- |
| 11620 |
// of the reduction because they may reorder intermediate operations. |
--- |
11620 |
// of the reduction because they may reorder intermediate operations. |
--- |
| 11621 |
unsigned CandidateBinOp = Op.getOpcode(); |
0 |
11621 |
unsigned CandidateBinOp = Op.getOpcode(); |
0 |
| 11622 |
if (Op.getValueType().isFloatingPoint()) { |
0 |
11622 |
if (Op.getValueType().isFloatingPoint()) { |
0 |
| 11623 |
SDNodeFlags Flags = Op->getFlags(); |
0 |
11623 |
SDNodeFlags Flags = Op->getFlags(); |
0 |
| 11624 |
switch (CandidateBinOp) { |
0 |
11624 |
switch (CandidateBinOp) { |
0 |
| 11625 |
case ISD::FADD: |
0 |
11625 |
case ISD::FADD: |
0 |
| 11626 |
if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation()) |
0 |
11626 |
if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation()) |
0 |
| 11627 |
return SDValue(); |
0 |
11627 |
return SDValue(); |
0 |
| 11628 |
break; |
0 |
11628 |
break; |
0 |
| 11629 |
default: |
0 |
11629 |
default: |
0 |
| 11630 |
llvm_unreachable("Unhandled FP opcode for binop reduction"); |
0 |
11630 |
llvm_unreachable("Unhandled FP opcode for binop reduction"); |
0 |
| 11631 |
} |
--- |
11631 |
} |
--- |
| 11632 |
} |
--- |
11632 |
} |
--- |
| 11633 |
|
--- |
11633 |
|
--- |
| 11634 |
// Matching failed - attempt to see if we did enough stages that a partial |
--- |
11634 |
// Matching failed - attempt to see if we did enough stages that a partial |
--- |
| 11635 |
// reduction from a subvector is possible. |
--- |
11635 |
// reduction from a subvector is possible. |
--- |
| 11636 |
auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) { |
0 |
11636 |
auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) { |
0 |
| 11637 |
if (!AllowPartials || !Op) |
0 |
11637 |
if (!AllowPartials || !Op) |
0 |
| 11638 |
return SDValue(); |
0 |
11638 |
return SDValue(); |
0 |
| 11639 |
EVT OpVT = Op.getValueType(); |
0 |
11639 |
EVT OpVT = Op.getValueType(); |
0 |
| 11640 |
EVT OpSVT = OpVT.getScalarType(); |
0 |
11640 |
EVT OpSVT = OpVT.getScalarType(); |
0 |
| 11641 |
EVT SubVT = EVT::getVectorVT(*getContext(), OpSVT, NumSubElts); |
0 |
11641 |
EVT SubVT = EVT::getVectorVT(*getContext(), OpSVT, NumSubElts); |
0 |
| 11642 |
if (!TLI->isExtractSubvectorCheap(SubVT, OpVT, 0)) |
0 |
11642 |
if (!TLI->isExtractSubvectorCheap(SubVT, OpVT, 0)) |
0 |
| 11643 |
return SDValue(); |
0 |
11643 |
return SDValue(); |
0 |
| 11644 |
BinOp = (ISD::NodeType)CandidateBinOp; |
0 |
11644 |
BinOp = (ISD::NodeType)CandidateBinOp; |
0 |
| 11645 |
return getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Op), SubVT, Op, |
0 |
11645 |
return getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Op), SubVT, Op, |
0 |
| 11646 |
getVectorIdxConstant(0, SDLoc(Op))); |
0 |
11646 |
getVectorIdxConstant(0, SDLoc(Op))); |
0 |
| 11647 |
}; |
0 |
11647 |
}; |
0 |
| 11648 |
|
--- |
11648 |
|
--- |
| 11649 |
// At each stage, we're looking for something that looks like: |
--- |
11649 |
// At each stage, we're looking for something that looks like: |
--- |
| 11650 |
// %s = shufflevector <8 x i32> %op, <8 x i32> undef, |
--- |
11650 |
// %s = shufflevector <8 x i32> %op, <8 x i32> undef, |
--- |
| 11651 |
// <8 x i32>
| --- |
11651 |
// <8 x i32>
| --- |
| |
| 11652 |
// i32 undef, i32 undef, i32 undef, i32 undef> |
--- |
11652 |
// i32 undef, i32 undef, i32 undef, i32 undef> |
--- |
| 11653 |
// %a = binop <8 x i32> %op, %s |
--- |
11653 |
// %a = binop <8 x i32> %op, %s |
--- |
| 11654 |
// Where the mask changes according to the stage. E.g. for a 3-stage pyramid, |
--- |
11654 |
// Where the mask changes according to the stage. E.g. for a 3-stage pyramid, |
--- |
| 11655 |
// we expect something like: |
--- |
11655 |
// we expect something like: |
--- |
| 11656 |
// <4,5,6,7,u,u,u,u> |
--- |
11656 |
// <4,5,6,7,u,u,u,u> |
--- |
| 11657 |
// <2,3,u,u,u,u,u,u> |
--- |
11657 |
// <2,3,u,u,u,u,u,u> |
--- |
| 11658 |
// <1,u,u,u,u,u,u,u> |
--- |
11658 |
// <1,u,u,u,u,u,u,u> |
--- |
| 11659 |
// While a partial reduction match would be: |
--- |
11659 |
// While a partial reduction match would be: |
--- |
| 11660 |
// <2,3,u,u,u,u,u,u> |
--- |
11660 |
// <2,3,u,u,u,u,u,u> |
--- |
| 11661 |
// <1,u,u,u,u,u,u,u> |
--- |
11661 |
// <1,u,u,u,u,u,u,u> |
--- |
| 11662 |
unsigned Stages = Log2_32(Op.getValueType().getVectorNumElements()); |
0 |
11662 |
unsigned Stages = Log2_32(Op.getValueType().getVectorNumElements()); |
0 |
| 11663 |
SDValue PrevOp; |
0 |
11663 |
SDValue PrevOp; |
0 |
| 11664 |
for (unsigned i = 0; i < Stages; ++i) { |
0 |
11664 |
for (unsigned i = 0; i < Stages; ++i) { |
0 |
| 11665 |
unsigned MaskEnd = (1 << i); |
0 |
11665 |
unsigned MaskEnd = (1 << i); |
0 |
| 11666 |
|
--- |
11666 |
|
--- |
| 11667 |
if (Op.getOpcode() != CandidateBinOp) |
0 |
11667 |
if (Op.getOpcode() != CandidateBinOp) |
0 |
| 11668 |
return PartialReduction(PrevOp, MaskEnd); |
0 |
11668 |
return PartialReduction(PrevOp, MaskEnd); |
0 |
| 11669 |
|
--- |
11669 |
|
--- |
| 11670 |
SDValue Op0 = Op.getOperand(0); |
0 |
11670 |
SDValue Op0 = Op.getOperand(0); |
0 |
| 11671 |
SDValue Op1 = Op.getOperand(1); |
0 |
11671 |
SDValue Op1 = Op.getOperand(1); |
0 |
| 11672 |
|
--- |
11672 |
|
--- |
| 11673 |
ShuffleVectorSDNode *Shuffle = dyn_cast(Op0); |
0 |
11673 |
ShuffleVectorSDNode *Shuffle = dyn_cast(Op0); |
0 |
| 11674 |
if (Shuffle) { |
0 |
11674 |
if (Shuffle) { |
0 |
| 11675 |
Op = Op1; |
0 |
11675 |
Op = Op1; |
0 |
| 11676 |
} else { |
--- |
11676 |
} else { |
--- |
| 11677 |
Shuffle = dyn_cast(Op1); |
0 |
11677 |
Shuffle = dyn_cast(Op1); |
0 |
| 11678 |
Op = Op0; |
0 |
11678 |
Op = Op0; |
0 |
| 11679 |
} |
--- |
11679 |
} |
--- |
| 11680 |
|
--- |
11680 |
|
--- |
| 11681 |
// The first operand of the shuffle should be the same as the other operand |
--- |
11681 |
// The first operand of the shuffle should be the same as the other operand |
--- |
| 11682 |
// of the binop. |
--- |
11682 |
// of the binop. |
--- |
| 11683 |
if (!Shuffle || Shuffle->getOperand(0) != Op) |
0 |
11683 |
if (!Shuffle || Shuffle->getOperand(0) != Op) |
0 |
| 11684 |
return PartialReduction(PrevOp, MaskEnd); |
0 |
11684 |
return PartialReduction(PrevOp, MaskEnd); |
0 |
| 11685 |
|
--- |
11685 |
|
--- |
| 11686 |
// Verify the shuffle has the expected (at this stage of the pyramid) mask. |
--- |
11686 |
// Verify the shuffle has the expected (at this stage of the pyramid) mask. |
--- |
| 11687 |
for (int Index = 0; Index < (int)MaskEnd; ++Index) |
0 |
11687 |
for (int Index = 0; Index < (int)MaskEnd; ++Index) |
0 |
| 11688 |
if (Shuffle->getMaskElt(Index) != (int)(MaskEnd + Index)) |
0 |
11688 |
if (Shuffle->getMaskElt(Index) != (int)(MaskEnd + Index)) |
0 |
| 11689 |
return PartialReduction(PrevOp, MaskEnd); |
0 |
11689 |
return PartialReduction(PrevOp, MaskEnd); |
0 |
| 11690 |
|
--- |
11690 |
|
--- |
| 11691 |
PrevOp = Op; |
0 |
11691 |
PrevOp = Op; |
0 |
| 11692 |
} |
--- |
11692 |
} |
--- |
| 11693 |
|
--- |
11693 |
|
--- |
| 11694 |
// Handle subvector reductions, which tend to appear after the shuffle |
--- |
11694 |
// Handle subvector reductions, which tend to appear after the shuffle |
--- |
| 11695 |
// reduction stages. |
--- |
11695 |
// reduction stages. |
--- |
| 11696 |
while (Op.getOpcode() == CandidateBinOp) { |
0 |
11696 |
while (Op.getOpcode() == CandidateBinOp) { |
0 |
| 11697 |
unsigned NumElts = Op.getValueType().getVectorNumElements(); |
0 |
11697 |
unsigned NumElts = Op.getValueType().getVectorNumElements(); |
0 |
| 11698 |
SDValue Op0 = Op.getOperand(0); |
0 |
11698 |
SDValue Op0 = Op.getOperand(0); |
0 |
| 11699 |
SDValue Op1 = Op.getOperand(1); |
0 |
11699 |
SDValue Op1 = Op.getOperand(1); |
0 |
| 11700 |
if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR || |
0 |
11700 |
if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR || |
0 |
| 11701 |
Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR || |
0 |
11701 |
Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR || |
0 |
| 11702 |
Op0.getOperand(0) != Op1.getOperand(0)) |
0 |
11702 |
Op0.getOperand(0) != Op1.getOperand(0)) |
0 |
| 11703 |
break; |
0 |
11703 |
break; |
0 |
| 11704 |
SDValue Src = Op0.getOperand(0); |
0 |
11704 |
SDValue Src = Op0.getOperand(0); |
0 |
| 11705 |
unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); |
0 |
11705 |
unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); |
0 |
| 11706 |
if (NumSrcElts != (2 * NumElts)) |
0 |
11706 |
if (NumSrcElts != (2 * NumElts)) |
0 |
| 11707 |
break; |
0 |
11707 |
break; |
0 |
| 11708 |
if (!(Op0.getConstantOperandAPInt(1) == 0 && |
0 |
11708 |
if (!(Op0.getConstantOperandAPInt(1) == 0 && |
0 |
| 11709 |
Op1.getConstantOperandAPInt(1) == NumElts) && |
0 |
11709 |
Op1.getConstantOperandAPInt(1) == NumElts) && |
0 |
| 11710 |
!(Op1.getConstantOperandAPInt(1) == 0 && |
0 |
11710 |
!(Op1.getConstantOperandAPInt(1) == 0 && |
0 |
| 11711 |
Op0.getConstantOperandAPInt(1) == NumElts)) |
0 |
11711 |
Op0.getConstantOperandAPInt(1) == NumElts)) |
0 |
| 11712 |
break; |
0 |
11712 |
break; |
0 |
| 11713 |
Op = Src; |
0 |
11713 |
Op = Src; |
0 |
| 11714 |
} |
--- |
11714 |
} |
--- |
| 11715 |
|
--- |
11715 |
|
--- |
| 11716 |
BinOp = (ISD::NodeType)CandidateBinOp; |
0 |
11716 |
BinOp = (ISD::NodeType)CandidateBinOp; |
0 |
| 11717 |
return Op; |
0 |
11717 |
return Op; |
0 |
| 11718 |
} |
--- |
11718 |
} |
--- |
| 11719 |
|
--- |
11719 |
|
--- |
| 11720 |
SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { |
0 |
11720 |
SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { |
0 |
| 11721 |
EVT VT = N->getValueType(0); |
0 |
11721 |
EVT VT = N->getValueType(0); |
0 |
| 11722 |
EVT EltVT = VT.getVectorElementType(); |
0 |
11722 |
EVT EltVT = VT.getVectorElementType(); |
0 |
| 11723 |
unsigned NE = VT.getVectorNumElements(); |
0 |
11723 |
unsigned NE = VT.getVectorNumElements(); |
0 |
| 11724 |
|
--- |
11724 |
|
--- |
| 11725 |
SDLoc dl(N); |
0 |
11725 |
SDLoc dl(N); |
0 |
| 11726 |
|
--- |
11726 |
|
--- |
| 11727 |
// If ResNE is 0, fully unroll the vector op. |
--- |
11727 |
// If ResNE is 0, fully unroll the vector op. |
--- |
| 11728 |
if (ResNE == 0) |
0 |
11728 |
if (ResNE == 0) |
0 |
| 11729 |
ResNE = NE; |
0 |
11729 |
ResNE = NE; |
0 |
| 11730 |
else if (NE > ResNE) |
0 |
11730 |
else if (NE > ResNE) |
0 |
| 11731 |
NE = ResNE; |
0 |
11731 |
NE = ResNE; |
0 |
| 11732 |
|
--- |
11732 |
|
--- |
| 11733 |
if (N->getNumValues() == 2) { |
0 |
11733 |
if (N->getNumValues() == 2) { |
0 |
| 11734 |
SmallVector Scalars0, Scalars1; |
0 |
11734 |
SmallVector Scalars0, Scalars1; |
0 |
| 11735 |
SmallVector Operands(N->getNumOperands()); |
0 |
11735 |
SmallVector Operands(N->getNumOperands()); |
0 |
| 11736 |
EVT VT1 = N->getValueType(1); |
0 |
11736 |
EVT VT1 = N->getValueType(1); |
0 |
| 11737 |
EVT EltVT1 = VT1.getVectorElementType(); |
0 |
11737 |
EVT EltVT1 = VT1.getVectorElementType(); |
0 |
| 11738 |
|
--- |
11738 |
|
--- |
| 11739 |
unsigned i; |
--- |
11739 |
unsigned i; |
--- |
| 11740 |
for (i = 0; i != NE; ++i) { |
0 |
11740 |
for (i = 0; i != NE; ++i) { |
0 |
| 11741 |
for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { |
0 |
11741 |
for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { |
0 |
| 11742 |
SDValue Operand = N->getOperand(j); |
0 |
11742 |
SDValue Operand = N->getOperand(j); |
0 |
| 11743 |
EVT OperandVT = Operand.getValueType(); |
0 |
11743 |
EVT OperandVT = Operand.getValueType(); |
0 |
| 11744 |
|
--- |
11744 |
|
--- |
| 11745 |
// A vector operand; extract a single element. |
--- |
11745 |
// A vector operand; extract a single element. |
--- |
| 11746 |
EVT OperandEltVT = OperandVT.getVectorElementType(); |
0 |
11746 |
EVT OperandEltVT = OperandVT.getVectorElementType(); |
0 |
| 11747 |
Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, |
0 |
11747 |
Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, |
0 |
| 11748 |
Operand, getVectorIdxConstant(i, dl)); |
--- |
11748 |
Operand, getVectorIdxConstant(i, dl)); |
--- |
| 11749 |
} |
--- |
11749 |
} |
--- |
| 11750 |
|
--- |
11750 |
|
--- |
| 11751 |
SDValue EltOp = getNode(N->getOpcode(), dl, {EltVT, EltVT1}, Operands); |
0 |
11751 |
SDValue EltOp = getNode(N->getOpcode(), dl, {EltVT, EltVT1}, Operands); |
0 |
| 11752 |
Scalars0.push_back(EltOp); |
0 |
11752 |
Scalars0.push_back(EltOp); |
0 |
| 11753 |
Scalars1.push_back(EltOp.getValue(1)); |
0 |
11753 |
Scalars1.push_back(EltOp.getValue(1)); |
0 |
| 11754 |
} |
--- |
11754 |
} |
--- |
| 11755 |
|
--- |
11755 |
|
--- |
| 11756 |
SDValue Vec0 = getBuildVector(VT, dl, Scalars0); |
0 |
11756 |
SDValue Vec0 = getBuildVector(VT, dl, Scalars0); |
0 |
| 11757 |
SDValue Vec1 = getBuildVector(VT1, dl, Scalars1); |
0 |
11757 |
SDValue Vec1 = getBuildVector(VT1, dl, Scalars1); |
0 |
| 11758 |
return getMergeValues({Vec0, Vec1}, dl); |
0 |
11758 |
return getMergeValues({Vec0, Vec1}, dl); |
0 |
| 11759 |
} |
0 |
11759 |
} |
0 |
| 11760 |
|
--- |
11760 |
|
--- |
| 11761 |
assert(N->getNumValues() == 1 && |
0 |
11761 |
assert(N->getNumValues() == 1 && |
0 |
| 11762 |
"Can't unroll a vector with multiple results!"); |
--- |
11762 |
"Can't unroll a vector with multiple results!"); |
--- |
| 11763 |
|
--- |
11763 |
|
--- |
| 11764 |
SmallVector Scalars; |
0 |
11764 |
SmallVector Scalars; |
0 |
| 11765 |
SmallVector Operands(N->getNumOperands()); |
0 |
11765 |
SmallVector Operands(N->getNumOperands()); |
0 |
| 11766 |
|
--- |
11766 |
|
--- |
| 11767 |
unsigned i; |
--- |
11767 |
unsigned i; |
--- |
| 11768 |
for (i= 0; i != NE; ++i) { |
0 |
11768 |
for (i= 0; i != NE; ++i) { |
0 |
| 11769 |
for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { |
0 |
11769 |
for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { |
0 |
| 11770 |
SDValue Operand = N->getOperand(j); |
0 |
11770 |
SDValue Operand = N->getOperand(j); |
0 |
| 11771 |
EVT OperandVT = Operand.getValueType(); |
0 |
11771 |
EVT OperandVT = Operand.getValueType(); |
0 |
| 11772 |
if (OperandVT.isVector()) { |
0 |
11772 |
if (OperandVT.isVector()) { |
0 |
| 11773 |
// A vector operand; extract a single element. |
--- |
11773 |
// A vector operand; extract a single element. |
--- |
| 11774 |
EVT OperandEltVT = OperandVT.getVectorElementType(); |
0 |
11774 |
EVT OperandEltVT = OperandVT.getVectorElementType(); |
0 |
| 11775 |
Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, |
0 |
11775 |
Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, |
0 |
| 11776 |
Operand, getVectorIdxConstant(i, dl)); |
--- |
11776 |
Operand, getVectorIdxConstant(i, dl)); |
--- |
| 11777 |
} else { |
--- |
11777 |
} else { |
--- |
| 11778 |
// A scalar operand; just use it as is. |
--- |
11778 |
// A scalar operand; just use it as is. |
--- |
| 11779 |
Operands[j] = Operand; |
0 |
11779 |
Operands[j] = Operand; |
0 |
| 11780 |
} |
--- |
11780 |
} |
--- |
| 11781 |
} |
--- |
11781 |
} |
--- |
| 11782 |
|
--- |
11782 |
|
--- |
| 11783 |
switch (N->getOpcode()) { |
0 |
11783 |
switch (N->getOpcode()) { |
0 |
| 11784 |
default: { |
0 |
11784 |
default: { |
0 |
| 11785 |
Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands, |
0 |
11785 |
Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands, |
0 |
| 11786 |
N->getFlags())); |
--- |
11786 |
N->getFlags())); |
--- |
| 11787 |
break; |
0 |
11787 |
break; |
0 |
| 11788 |
} |
--- |
11788 |
} |
--- |
| 11789 |
case ISD::VSELECT: |
0 |
11789 |
case ISD::VSELECT: |
0 |
| 11790 |
Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands)); |
0 |
11790 |
Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands)); |
0 |
| 11791 |
break; |
0 |
11791 |
break; |
0 |
| 11792 |
case ISD::SHL: |
0 |
11792 |
case ISD::SHL: |
0 |
| 11793 |
case ISD::SRA: |
--- |
11793 |
case ISD::SRA: |
--- |
| 11794 |
case ISD::SRL: |
--- |
11794 |
case ISD::SRL: |
--- |
| 11795 |
case ISD::ROTL: |
--- |
11795 |
case ISD::ROTL: |
--- |
| 11796 |
case ISD::ROTR: |
--- |
11796 |
case ISD::ROTR: |
--- |
| 11797 |
Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], |
0 |
11797 |
Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], |
0 |
| 11798 |
getShiftAmountOperand(Operands[0].getValueType(), |
0 |
11798 |
getShiftAmountOperand(Operands[0].getValueType(), |
0 |
| 11799 |
Operands[1]))); |
0 |
11799 |
Operands[1]))); |
0 |
| 11800 |
break; |
0 |
11800 |
break; |
0 |
| 11801 |
case ISD::SIGN_EXTEND_INREG: { |
0 |
11801 |
case ISD::SIGN_EXTEND_INREG: { |
0 |
| 11802 |
EVT ExtVT = cast(Operands[1])->getVT().getVectorElementType(); |
0 |
11802 |
EVT ExtVT = cast(Operands[1])->getVT().getVectorElementType(); |
0 |
| 11803 |
Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, |
0 |
11803 |
Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, |
0 |
| 11804 |
Operands[0], |
0 |
11804 |
Operands[0], |
0 |
| 11805 |
getValueType(ExtVT))); |
--- |
11805 |
getValueType(ExtVT))); |
--- |
| 11806 |
} |
--- |
11806 |
} |
--- |
| 11807 |
} |
--- |
11807 |
} |
--- |
| 11808 |
} |
--- |
11808 |
} |
--- |
| 11809 |
|
--- |
11809 |
|
--- |
| 11810 |
for (; i < ResNE; ++i) |
0 |
11810 |
for (; i < ResNE; ++i) |
0 |
| 11811 |
Scalars.push_back(getUNDEF(EltVT)); |
0 |
11811 |
Scalars.push_back(getUNDEF(EltVT)); |
0 |
| 11812 |
|
--- |
11812 |
|
--- |
| 11813 |
EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE); |
0 |
11813 |
EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE); |
0 |
| 11814 |
return getBuildVector(VecVT, dl, Scalars); |
0 |
11814 |
return getBuildVector(VecVT, dl, Scalars); |
0 |
| 11815 |
} |
0 |
11815 |
} |
0 |
| 11816 |
|
--- |
11816 |
|
--- |
| 11817 |
std::pair SelectionDAG::UnrollVectorOverflowOp( |
0 |
11817 |
std::pair SelectionDAG::UnrollVectorOverflowOp( |
0 |
| 11818 |
SDNode *N, unsigned ResNE) { |
--- |
11818 |
SDNode *N, unsigned ResNE) { |
--- |
| 11819 |
unsigned Opcode = N->getOpcode(); |
0 |
11819 |
unsigned Opcode = N->getOpcode(); |
0 |
| 11820 |
assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO || |
0 |
11820 |
assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO || |
0 |
| 11821 |
Opcode == ISD::USUBO || Opcode == ISD::SSUBO || |
--- |
11821 |
Opcode == ISD::USUBO || Opcode == ISD::SSUBO || |
--- |
| 11822 |
Opcode == ISD::UMULO || Opcode == ISD::SMULO) && |
--- |
11822 |
Opcode == ISD::UMULO || Opcode == ISD::SMULO) && |
--- |
| 11823 |
"Expected an overflow opcode"); |
--- |
11823 |
"Expected an overflow opcode"); |
--- |
| 11824 |
|
--- |
11824 |
|
--- |
| 11825 |
EVT ResVT = N->getValueType(0); |
0 |
11825 |
EVT ResVT = N->getValueType(0); |
0 |
| 11826 |
EVT OvVT = N->getValueType(1); |
0 |
11826 |
EVT OvVT = N->getValueType(1); |
0 |
| 11827 |
EVT ResEltVT = ResVT.getVectorElementType(); |
0 |
11827 |
EVT ResEltVT = ResVT.getVectorElementType(); |
0 |
| 11828 |
EVT OvEltVT = OvVT.getVectorElementType(); |
0 |
11828 |
EVT OvEltVT = OvVT.getVectorElementType(); |
0 |
| 11829 |
SDLoc dl(N); |
0 |
11829 |
SDLoc dl(N); |
0 |
| 11830 |
|
--- |
11830 |
|
--- |
| 11831 |
// If ResNE is 0, fully unroll the vector op. |
--- |
11831 |
// If ResNE is 0, fully unroll the vector op. |
--- |
| 11832 |
unsigned NE = ResVT.getVectorNumElements(); |
0 |
11832 |
unsigned NE = ResVT.getVectorNumElements(); |
0 |
| 11833 |
if (ResNE == 0) |
0 |
11833 |
if (ResNE == 0) |
0 |
| 11834 |
ResNE = NE; |
0 |
11834 |
ResNE = NE; |
0 |
| 11835 |
else if (NE > ResNE) |
0 |
11835 |
else if (NE > ResNE) |
0 |
| 11836 |
NE = ResNE; |
0 |
11836 |
NE = ResNE; |
0 |
| 11837 |
|
--- |
11837 |
|
--- |
| 11838 |
SmallVector LHSScalars; |
0 |
11838 |
SmallVector LHSScalars; |
0 |
| 11839 |
SmallVector RHSScalars; |
0 |
11839 |
SmallVector RHSScalars; |
0 |
| 11840 |
ExtractVectorElements(N->getOperand(0), LHSScalars, 0, NE); |
0 |
11840 |
ExtractVectorElements(N->getOperand(0), LHSScalars, 0, NE); |
0 |
| 11841 |
ExtractVectorElements(N->getOperand(1), RHSScalars, 0, NE); |
0 |
11841 |
ExtractVectorElements(N->getOperand(1), RHSScalars, 0, NE); |
0 |
| 11842 |
|
--- |
11842 |
|
--- |
| 11843 |
EVT SVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), ResEltVT); |
0 |
11843 |
EVT SVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), ResEltVT); |
0 |
| 11844 |
SDVTList VTs = getVTList(ResEltVT, SVT); |
0 |
11844 |
SDVTList VTs = getVTList(ResEltVT, SVT); |
0 |
| 11845 |
SmallVector ResScalars; |
0 |
11845 |
SmallVector ResScalars; |
0 |
| 11846 |
SmallVector OvScalars; |
0 |
11846 |
SmallVector OvScalars; |
0 |
| 11847 |
for (unsigned i = 0; i < NE; ++i) { |
0 |
11847 |
for (unsigned i = 0; i < NE; ++i) { |
0 |
| 11848 |
SDValue Res = getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]); |
0 |
11848 |
SDValue Res = getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]); |
0 |
| 11849 |
SDValue Ov = |
--- |
11849 |
SDValue Ov = |
--- |
| 11850 |
getSelect(dl, OvEltVT, Res.getValue(1), |
0 |
11850 |
getSelect(dl, OvEltVT, Res.getValue(1), |
0 |
| 11851 |
getBoolConstant(true, dl, OvEltVT, ResVT), |
--- |
11851 |
getBoolConstant(true, dl, OvEltVT, ResVT), |
--- |
| 11852 |
getConstant(0, dl, OvEltVT)); |
--- |
11852 |
getConstant(0, dl, OvEltVT)); |
--- |
| 11853 |
|
--- |
11853 |
|
--- |
| 11854 |
ResScalars.push_back(Res); |
0 |
11854 |
ResScalars.push_back(Res); |
0 |
| 11855 |
OvScalars.push_back(Ov); |
0 |
11855 |
OvScalars.push_back(Ov); |
0 |
| 11856 |
} |
--- |
11856 |
} |
--- |
| 11857 |
|
--- |
11857 |
|
--- |
| 11858 |
ResScalars.append(ResNE - NE, getUNDEF(ResEltVT)); |
0 |
11858 |
ResScalars.append(ResNE - NE, getUNDEF(ResEltVT)); |
0 |
| 11859 |
OvScalars.append(ResNE - NE, getUNDEF(OvEltVT)); |
0 |
11859 |
OvScalars.append(ResNE - NE, getUNDEF(OvEltVT)); |
0 |
| 11860 |
|
--- |
11860 |
|
--- |
| 11861 |
EVT NewResVT = EVT::getVectorVT(*getContext(), ResEltVT, ResNE); |
0 |
11861 |
EVT NewResVT = EVT::getVectorVT(*getContext(), ResEltVT, ResNE); |
0 |
| 11862 |
EVT NewOvVT = EVT::getVectorVT(*getContext(), OvEltVT, ResNE); |
0 |
11862 |
EVT NewOvVT = EVT::getVectorVT(*getContext(), OvEltVT, ResNE); |
0 |
| 11863 |
return std::make_pair(getBuildVector(NewResVT, dl, ResScalars), |
0 |
11863 |
return std::make_pair(getBuildVector(NewResVT, dl, ResScalars), |
0 |
| 11864 |
getBuildVector(NewOvVT, dl, OvScalars)); |
0 |
11864 |
getBuildVector(NewOvVT, dl, OvScalars)); |
0 |
| 11865 |
} |
0 |
11865 |
} |
0 |
| 11866 |
|
--- |
11866 |
|
--- |
| 11867 |
bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD, |
0 |
11867 |
bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD, |
0 |
| 11868 |
LoadSDNode *Base, |
--- |
11868 |
LoadSDNode *Base, |
--- |
| 11869 |
unsigned Bytes, |
--- |
11869 |
unsigned Bytes, |
--- |
| 11870 |
int Dist) const { |
--- |
11870 |
int Dist) const { |
--- |
| 11871 |
if (LD->isVolatile() || Base->isVolatile()) |
0 |
11871 |
if (LD->isVolatile() || Base->isVolatile()) |
0 |
| 11872 |
return false; |
0 |
11872 |
return false; |
0 |
| 11873 |
// TODO: probably too restrictive for atomics, revisit |
--- |
11873 |
// TODO: probably too restrictive for atomics, revisit |
--- |
| 11874 |
if (!LD->isSimple()) |
0 |
11874 |
if (!LD->isSimple()) |
0 |
| 11875 |
return false; |
0 |
11875 |
return false; |
0 |
| 11876 |
if (LD->isIndexed() || Base->isIndexed()) |
0 |
11876 |
if (LD->isIndexed() || Base->isIndexed()) |
0 |
| 11877 |
return false; |
0 |
11877 |
return false; |
0 |
| 11878 |
if (LD->getChain() != Base->getChain()) |
0 |
11878 |
if (LD->getChain() != Base->getChain()) |
0 |
| 11879 |
return false; |
0 |
11879 |
return false; |
0 |
| 11880 |
EVT VT = LD->getMemoryVT(); |
0 |
11880 |
EVT VT = LD->getMemoryVT(); |
0 |
| 11881 |
if (VT.getSizeInBits() / 8 != Bytes) |
0 |
11881 |
if (VT.getSizeInBits() / 8 != Bytes) |
0 |
| 11882 |
return false; |
0 |
11882 |
return false; |
0 |
| 11883 |
|
--- |
11883 |
|
--- |
| 11884 |
auto BaseLocDecomp = BaseIndexOffset::match(Base, *this); |
0 |
11884 |
auto BaseLocDecomp = BaseIndexOffset::match(Base, *this); |
0 |
| 11885 |
auto LocDecomp = BaseIndexOffset::match(LD, *this); |
0 |
11885 |
auto LocDecomp = BaseIndexOffset::match(LD, *this); |
0 |
| 11886 |
|
--- |
11886 |
|
--- |
| 11887 |
int64_t Offset = 0; |
0 |
11887 |
int64_t Offset = 0; |
0 |
| 11888 |
if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset)) |
0 |
11888 |
if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset)) |
0 |
| 11889 |
return (Dist * (int64_t)Bytes == Offset); |
0 |
11889 |
return (Dist * (int64_t)Bytes == Offset); |
0 |
| 11890 |
return false; |
0 |
11890 |
return false; |
0 |
| 11891 |
} |
--- |
11891 |
} |
--- |
| 11892 |
|
--- |
11892 |
|
--- |
| 11893 |
/// InferPtrAlignment - Infer alignment of a load / store address. Return |
--- |
11893 |
/// InferPtrAlignment - Infer alignment of a load / store address. Return |
--- |
| 11894 |
/// std::nullopt if it cannot be inferred. |
--- |
11894 |
/// std::nullopt if it cannot be inferred. |
--- |
| 11895 |
MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const { |
0 |
11895 |
MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const { |
0 |
| 11896 |
// If this is a GlobalAddress + cst, return the alignment. |
--- |
11896 |
// If this is a GlobalAddress + cst, return the alignment. |
--- |
| 11897 |
const GlobalValue *GV = nullptr; |
0 |
11897 |
const GlobalValue *GV = nullptr; |
0 |
| 11898 |
int64_t GVOffset = 0; |
0 |
11898 |
int64_t GVOffset = 0; |
0 |
| 11899 |
if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { |
0 |
11899 |
if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { |
0 |
| 11900 |
unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType()); |
0 |
11900 |
unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType()); |
0 |
| 11901 |
KnownBits Known(PtrWidth); |
0 |
11901 |
KnownBits Known(PtrWidth); |
0 |
| 11902 |
llvm::computeKnownBits(GV, Known, getDataLayout()); |
0 |
11902 |
llvm::computeKnownBits(GV, Known, getDataLayout()); |
0 |
| 11903 |
unsigned AlignBits = Known.countMinTrailingZeros(); |
0 |
11903 |
unsigned AlignBits = Known.countMinTrailingZeros(); |
0 |
| 11904 |
if (AlignBits) |
0 |
11904 |
if (AlignBits) |
0 |
| 11905 |
return commonAlignment(Align(1ull << std::min(31U, AlignBits)), GVOffset); |
0 |
11905 |
return commonAlignment(Align(1ull << std::min(31U, AlignBits)), GVOffset); |
0 |
| 11906 |
} |
0 |
11906 |
} |
0 |
| 11907 |
|
--- |
11907 |
|
--- |
| 11908 |
// If this is a direct reference to a stack slot, use information about the |
--- |
11908 |
// If this is a direct reference to a stack slot, use information about the |
--- |
| 11909 |
// stack slot's alignment. |
--- |
11909 |
// stack slot's alignment. |
--- |
| 11910 |
int FrameIdx = INT_MIN; |
0 |
11910 |
int FrameIdx = INT_MIN; |
0 |
| 11911 |
int64_t FrameOffset = 0; |
0 |
11911 |
int64_t FrameOffset = 0; |
0 |
| 11912 |
if (FrameIndexSDNode *FI = dyn_cast(Ptr)) { |
0 |
11912 |
if (FrameIndexSDNode *FI = dyn_cast(Ptr)) { |
0 |
| 11913 |
FrameIdx = FI->getIndex(); |
0 |
11913 |
FrameIdx = FI->getIndex(); |
0 |
| 11914 |
} else if (isBaseWithConstantOffset(Ptr) && |
0 |
11914 |
} else if (isBaseWithConstantOffset(Ptr) && |
0 |
| 11915 |
isa(Ptr.getOperand(0))) { |
0 |
11915 |
isa(Ptr.getOperand(0))) { |
0 |
| 11916 |
// Handle FI+Cst |
--- |
11916 |
// Handle FI+Cst |
--- |
| 11917 |
FrameIdx = cast(Ptr.getOperand(0))->getIndex(); |
0 |
11917 |
FrameIdx = cast(Ptr.getOperand(0))->getIndex(); |
0 |
| 11918 |
FrameOffset = Ptr.getConstantOperandVal(1); |
0 |
11918 |
FrameOffset = Ptr.getConstantOperandVal(1); |
0 |
| 11919 |
} |
--- |
11919 |
} |
--- |
| 11920 |
|
--- |
11920 |
|
--- |
| 11921 |
if (FrameIdx != INT_MIN) { |
0 |
11921 |
if (FrameIdx != INT_MIN) { |
0 |
| 11922 |
const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo(); |
0 |
11922 |
const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo(); |
0 |
| 11923 |
return commonAlignment(MFI.getObjectAlign(FrameIdx), FrameOffset); |
0 |
11923 |
return commonAlignment(MFI.getObjectAlign(FrameIdx), FrameOffset); |
0 |
| 11924 |
} |
--- |
11924 |
} |
--- |
| 11925 |
|
--- |
11925 |
|
--- |
| 11926 |
return std::nullopt; |
0 |
11926 |
return std::nullopt; |
0 |
| 11927 |
} |
--- |
11927 |
} |
--- |
| 11928 |
|
--- |
11928 |
|
--- |
| 11929 |
/// Split the scalar node with EXTRACT_ELEMENT using the provided |
--- |
11929 |
/// Split the scalar node with EXTRACT_ELEMENT using the provided |
--- |
| 11930 |
/// VTs and return the low/high part. |
--- |
11930 |
/// VTs and return the low/high part. |
--- |
| 11931 |
std::pair SelectionDAG::SplitScalar(const SDValue &N, |
0 |
11931 |
std::pair SelectionDAG::SplitScalar(const SDValue &N, |
0 |
| 11932 |
const SDLoc &DL, |
--- |
11932 |
const SDLoc &DL, |
--- |
| 11933 |
const EVT &LoVT, |
--- |
11933 |
const EVT &LoVT, |
--- |
| 11934 |
const EVT &HiVT) { |
--- |
11934 |
const EVT &HiVT) { |
--- |
| 11935 |
assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() && |
0 |
11935 |
assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() && |
0 |
| 11936 |
"Split node must be a scalar type"); |
--- |
11936 |
"Split node must be a scalar type"); |
--- |
| 11937 |
SDValue Lo = |
--- |
11937 |
SDValue Lo = |
--- |
| 11938 |
getNode(ISD::EXTRACT_ELEMENT, DL, LoVT, N, getIntPtrConstant(0, DL)); |
0 |
11938 |
getNode(ISD::EXTRACT_ELEMENT, DL, LoVT, N, getIntPtrConstant(0, DL)); |
0 |
| 11939 |
SDValue Hi = |
--- |
11939 |
SDValue Hi = |
--- |
| 11940 |
getNode(ISD::EXTRACT_ELEMENT, DL, HiVT, N, getIntPtrConstant(1, DL)); |
0 |
11940 |
getNode(ISD::EXTRACT_ELEMENT, DL, HiVT, N, getIntPtrConstant(1, DL)); |
0 |
| 11941 |
return std::make_pair(Lo, Hi); |
0 |
11941 |
return std::make_pair(Lo, Hi); |
0 |
| 11942 |
} |
--- |
11942 |
} |
--- |
| 11943 |
|
--- |
11943 |
|
--- |
| 11944 |
/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type |
--- |
11944 |
/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type |
--- |
| 11945 |
/// which is split (or expanded) into two not necessarily identical pieces. |
--- |
11945 |
/// which is split (or expanded) into two not necessarily identical pieces. |
--- |
| 11946 |
std::pair SelectionDAG::GetSplitDestVTs(const EVT &VT) const { |
0 |
11946 |
std::pair SelectionDAG::GetSplitDestVTs(const EVT &VT) const { |
0 |
| 11947 |
// Currently all types are split in half. |
--- |
11947 |
// Currently all types are split in half. |
--- |
| 11948 |
EVT LoVT, HiVT; |
0 |
11948 |
EVT LoVT, HiVT; |
0 |
| 11949 |
if (!VT.isVector()) |
0 |
11949 |
if (!VT.isVector()) |
0 |
| 11950 |
LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT); |
0 |
11950 |
LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT); |
0 |
| 11951 |
else |
--- |
11951 |
else |
--- |
| 11952 |
LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext()); |
0 |
11952 |
LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext()); |
0 |
| 11953 |
|
--- |
11953 |
|
--- |
| 11954 |
return std::make_pair(LoVT, HiVT); |
0 |
11954 |
return std::make_pair(LoVT, HiVT); |
0 |
| 11955 |
} |
--- |
11955 |
} |
--- |
| 11956 |
|
--- |
11956 |
|
--- |
| 11957 |
/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a |
--- |
11957 |
/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a |
--- |
| 11958 |
/// type, dependent on an enveloping VT that has been split into two identical |
--- |
11958 |
/// type, dependent on an enveloping VT that has been split into two identical |
--- |
| 11959 |
/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size. |
--- |
11959 |
/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size. |
--- |
| 11960 |
std::pair |
--- |
11960 |
std::pair |
--- |
| 11961 |
SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, |
0 |
11961 |
SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, |
0 |
| 11962 |
bool *HiIsEmpty) const { |
--- |
11962 |
bool *HiIsEmpty) const { |
--- |
| 11963 |
EVT EltTp = VT.getVectorElementType(); |
0 |
11963 |
EVT EltTp = VT.getVectorElementType(); |
0 |
| 11964 |
// Examples: |
--- |
11964 |
// Examples: |
--- |
| 11965 |
// custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty) |
--- |
11965 |
// custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty) |
--- |
| 11966 |
// custom VL=9 with enveloping VL=8/8 yields 8/1 |
--- |
11966 |
// custom VL=9 with enveloping VL=8/8 yields 8/1 |
--- |
| 11967 |
// custom VL=10 with enveloping VL=8/8 yields 8/2 |
--- |
11967 |
// custom VL=10 with enveloping VL=8/8 yields 8/2 |
--- |
| 11968 |
// etc. |
--- |
11968 |
// etc. |
--- |
| 11969 |
ElementCount VTNumElts = VT.getVectorElementCount(); |
0 |
11969 |
ElementCount VTNumElts = VT.getVectorElementCount(); |
0 |
| 11970 |
ElementCount EnvNumElts = EnvVT.getVectorElementCount(); |
0 |
11970 |
ElementCount EnvNumElts = EnvVT.getVectorElementCount(); |
0 |
| 11971 |
assert(VTNumElts.isScalable() == EnvNumElts.isScalable() && |
0 |
11971 |
assert(VTNumElts.isScalable() == EnvNumElts.isScalable() && |
0 |
| 11972 |
"Mixing fixed width and scalable vectors when enveloping a type"); |
--- |
11972 |
"Mixing fixed width and scalable vectors when enveloping a type"); |
--- |
| 11973 |
EVT LoVT, HiVT; |
0 |
11973 |
EVT LoVT, HiVT; |
0 |
| 11974 |
if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) { |
0 |
11974 |
if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) { |
0 |
| 11975 |
LoVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts); |
0 |
11975 |
LoVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts); |
0 |
| 11976 |
HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts); |
0 |
11976 |
HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts); |
0 |
| 11977 |
*HiIsEmpty = false; |
0 |
11977 |
*HiIsEmpty = false; |
0 |
| 11978 |
} else { |
--- |
11978 |
} else { |
--- |
| 11979 |
// Flag that hi type has zero storage size, but return split envelop type |
--- |
11979 |
// Flag that hi type has zero storage size, but return split envelop type |
--- |
| 11980 |
// (this would be easier if vector types with zero elements were allowed). |
--- |
11980 |
// (this would be easier if vector types with zero elements were allowed). |
--- |
| 11981 |
LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts); |
0 |
11981 |
LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts); |
0 |
| 11982 |
HiVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts); |
0 |
11982 |
HiVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts); |
0 |
| 11983 |
*HiIsEmpty = true; |
0 |
11983 |
*HiIsEmpty = true; |
0 |
| 11984 |
} |
--- |
11984 |
} |
--- |
| 11985 |
return std::make_pair(LoVT, HiVT); |
0 |
11985 |
return std::make_pair(LoVT, HiVT); |
0 |
| 11986 |
} |
--- |
11986 |
} |
--- |
| 11987 |
|
--- |
11987 |
|
--- |
| 11988 |
/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the |
--- |
11988 |
/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the |
--- |
| 11989 |
/// low/high part. |
--- |
11989 |
/// low/high part. |
--- |
| 11990 |
std::pair |
--- |
11990 |
std::pair |
--- |
| 11991 |
SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, |
0 |
11991 |
SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, |
0 |
| 11992 |
const EVT &HiVT) { |
--- |
11992 |
const EVT &HiVT) { |
--- |
| 11993 |
assert(LoVT.isScalableVector() == HiVT.isScalableVector() && |
0 |
11993 |
assert(LoVT.isScalableVector() == HiVT.isScalableVector() && |
0 |
| 11994 |
LoVT.isScalableVector() == N.getValueType().isScalableVector() && |
--- |
11994 |
LoVT.isScalableVector() == N.getValueType().isScalableVector() && |
--- |
| 11995 |
"Splitting vector with an invalid mixture of fixed and scalable " |
--- |
11995 |
"Splitting vector with an invalid mixture of fixed and scalable " |
--- |
| 11996 |
"vector types"); |
--- |
11996 |
"vector types"); |
--- |
| 11997 |
assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <= |
0 |
11997 |
assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <= |
0 |
| 11998 |
N.getValueType().getVectorMinNumElements() && |
--- |
11998 |
N.getValueType().getVectorMinNumElements() && |
--- |
| 11999 |
"More vector elements requested than available!"); |
--- |
11999 |
"More vector elements requested than available!"); |
--- |
| 12000 |
SDValue Lo, Hi; |
0 |
12000 |
SDValue Lo, Hi; |
0 |
| 12001 |
Lo = |
0 |
12001 |
Lo = |
0 |
| 12002 |
getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, getVectorIdxConstant(0, DL)); |
0 |
12002 |
getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, getVectorIdxConstant(0, DL)); |
0 |
| 12003 |
// For scalable vectors it is safe to use LoVT.getVectorMinNumElements() |
--- |
12003 |
// For scalable vectors it is safe to use LoVT.getVectorMinNumElements() |
--- |
| 12004 |
// (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales |
--- |
12004 |
// (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales |
--- |
| 12005 |
// IDX with the runtime scaling factor of the result vector type. For |
--- |
12005 |
// IDX with the runtime scaling factor of the result vector type. For |
--- |
| 12006 |
// fixed-width result vectors, that runtime scaling factor is 1. |
--- |
12006 |
// fixed-width result vectors, that runtime scaling factor is 1. |
--- |
| 12007 |
Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N, |
0 |
12007 |
Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N, |
0 |
| 12008 |
getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL)); |
0 |
12008 |
getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL)); |
0 |
| 12009 |
return std::make_pair(Lo, Hi); |
0 |
12009 |
return std::make_pair(Lo, Hi); |
0 |
| 12010 |
} |
--- |
12010 |
} |
--- |
| 12011 |
|
--- |
12011 |
|
--- |
| 12012 |
std::pair SelectionDAG::SplitEVL(SDValue N, EVT VecVT, |
0 |
12012 |
std::pair SelectionDAG::SplitEVL(SDValue N, EVT VecVT, |
0 |
| 12013 |
const SDLoc &DL) { |
--- |
12013 |
const SDLoc &DL) { |
--- |
| 12014 |
// Split the vector length parameter. |
--- |
12014 |
// Split the vector length parameter. |
--- |
| 12015 |
// %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts). |
--- |
12015 |
// %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts). |
--- |
| 12016 |
EVT VT = N.getValueType(); |
0 |
12016 |
EVT VT = N.getValueType(); |
0 |
| 12017 |
assert(VecVT.getVectorElementCount().isKnownEven() && |
0 |
12017 |
assert(VecVT.getVectorElementCount().isKnownEven() && |
0 |
| 12018 |
"Expecting the mask to be an evenly-sized vector"); |
--- |
12018 |
"Expecting the mask to be an evenly-sized vector"); |
--- |
| 12019 |
unsigned HalfMinNumElts = VecVT.getVectorMinNumElements() / 2; |
0 |
12019 |
unsigned HalfMinNumElts = VecVT.getVectorMinNumElements() / 2; |
0 |
| 12020 |
SDValue HalfNumElts = |
--- |
12020 |
SDValue HalfNumElts = |
--- |
| 12021 |
VecVT.isFixedLengthVector() |
0 |
12021 |
VecVT.isFixedLengthVector() |
0 |
| 12022 |
? getConstant(HalfMinNumElts, DL, VT) |
0 |
12022 |
? getConstant(HalfMinNumElts, DL, VT) |
0 |
| 12023 |
: getVScale(DL, VT, APInt(VT.getScalarSizeInBits(), HalfMinNumElts)); |
0 |
12023 |
: getVScale(DL, VT, APInt(VT.getScalarSizeInBits(), HalfMinNumElts)); |
0 |
| 12024 |
SDValue Lo = getNode(ISD::UMIN, DL, VT, N, HalfNumElts); |
0 |
12024 |
SDValue Lo = getNode(ISD::UMIN, DL, VT, N, HalfNumElts); |
0 |
| 12025 |
SDValue Hi = getNode(ISD::USUBSAT, DL, VT, N, HalfNumElts); |
0 |
12025 |
SDValue Hi = getNode(ISD::USUBSAT, DL, VT, N, HalfNumElts); |
0 |
| 12026 |
return std::make_pair(Lo, Hi); |
0 |
12026 |
return std::make_pair(Lo, Hi); |
0 |
| 12027 |
} |
--- |
12027 |
} |
--- |
| 12028 |
|
--- |
12028 |
|
--- |
| 12029 |
/// Widen the vector up to the next power of two using INSERT_SUBVECTOR. |
--- |
12029 |
/// Widen the vector up to the next power of two using INSERT_SUBVECTOR. |
--- |
| 12030 |
SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) { |
0 |
12030 |
SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) { |
0 |
| 12031 |
EVT VT = N.getValueType(); |
0 |
12031 |
EVT VT = N.getValueType(); |
0 |
| 12032 |
EVT WideVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(), |
0 |
12032 |
EVT WideVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(), |
0 |
| 12033 |
NextPowerOf2(VT.getVectorNumElements())); |
0 |
12033 |
NextPowerOf2(VT.getVectorNumElements())); |
0 |
| 12034 |
return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N, |
0 |
12034 |
return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N, |
0 |
| 12035 |
getVectorIdxConstant(0, DL)); |
0 |
12035 |
getVectorIdxConstant(0, DL)); |
0 |
| 12036 |
} |
--- |
12036 |
} |
--- |
| 12037 |
|
--- |
12037 |
|
--- |
| 12038 |
void SelectionDAG::ExtractVectorElements(SDValue Op, |
0 |
12038 |
void SelectionDAG::ExtractVectorElements(SDValue Op, |
0 |
| 12039 |
SmallVectorImpl &Args, |
--- |
12039 |
SmallVectorImpl &Args, |
--- |
| 12040 |
unsigned Start, unsigned Count, |
--- |
12040 |
unsigned Start, unsigned Count, |
--- |
| 12041 |
EVT EltVT) { |
--- |
12041 |
EVT EltVT) { |
--- |
| 12042 |
EVT VT = Op.getValueType(); |
0 |
12042 |
EVT VT = Op.getValueType(); |
0 |
| 12043 |
if (Count == 0) |
0 |
12043 |
if (Count == 0) |
0 |
| 12044 |
Count = VT.getVectorNumElements(); |
0 |
12044 |
Count = VT.getVectorNumElements(); |
0 |
| 12045 |
if (EltVT == EVT()) |
0 |
12045 |
if (EltVT == EVT()) |
0 |
| 12046 |
EltVT = VT.getVectorElementType(); |
0 |
12046 |
EltVT = VT.getVectorElementType(); |
0 |
| 12047 |
SDLoc SL(Op); |
0 |
12047 |
SDLoc SL(Op); |
0 |
| 12048 |
for (unsigned i = Start, e = Start + Count; i != e; ++i) { |
0 |
12048 |
for (unsigned i = Start, e = Start + Count; i != e; ++i) { |
0 |
| 12049 |
Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Op, |
0 |
12049 |
Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Op, |
0 |
| 12050 |
getVectorIdxConstant(i, SL))); |
--- |
12050 |
getVectorIdxConstant(i, SL))); |
--- |
| 12051 |
} |
--- |
12051 |
} |
--- |
| 12052 |
} |
0 |
12052 |
} |
0 |
| 12053 |
|
--- |
12053 |
|
--- |
| 12054 |
// getAddressSpace - Return the address space this GlobalAddress belongs to. |
--- |
12054 |
// getAddressSpace - Return the address space this GlobalAddress belongs to. |
--- |
| 12055 |
unsigned GlobalAddressSDNode::getAddressSpace() const { |
0 |
12055 |
unsigned GlobalAddressSDNode::getAddressSpace() const { |
0 |
| 12056 |
return getGlobal()->getType()->getAddressSpace(); |
0 |
12056 |
return getGlobal()->getType()->getAddressSpace(); |
0 |
| 12057 |
} |
--- |
12057 |
} |
--- |
| 12058 |
|
--- |
12058 |
|
--- |
| 12059 |
Type *ConstantPoolSDNode::getType() const { |
0 |
12059 |
Type *ConstantPoolSDNode::getType() const { |
0 |
| 12060 |
if (isMachineConstantPoolEntry()) |
0 |
12060 |
if (isMachineConstantPoolEntry()) |
0 |
| 12061 |
return Val.MachineCPVal->getType(); |
0 |
12061 |
return Val.MachineCPVal->getType(); |
0 |
| 12062 |
return Val.ConstVal->getType(); |
0 |
12062 |
return Val.ConstVal->getType(); |
0 |
| 12063 |
} |
--- |
12063 |
} |
--- |
| 12064 |
|
--- |
12064 |
|
--- |
| 12065 |
bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef, |
0 |
12065 |
bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef, |
0 |
| 12066 |
unsigned &SplatBitSize, |
--- |
12066 |
unsigned &SplatBitSize, |
--- |
| 12067 |
bool &HasAnyUndefs, |
--- |
12067 |
bool &HasAnyUndefs, |
--- |
| 12068 |
unsigned MinSplatBits, |
--- |
12068 |
unsigned MinSplatBits, |
--- |
| 12069 |
bool IsBigEndian) const { |
--- |
12069 |
bool IsBigEndian) const { |
--- |
| 12070 |
EVT VT = getValueType(0); |
0 |
12070 |
EVT VT = getValueType(0); |
0 |
| 12071 |
assert(VT.isVector() && "Expected a vector type"); |
0 |
12071 |
assert(VT.isVector() && "Expected a vector type"); |
0 |
| 12072 |
unsigned VecWidth = VT.getSizeInBits(); |
0 |
12072 |
unsigned VecWidth = VT.getSizeInBits(); |
0 |
| 12073 |
if (MinSplatBits > VecWidth) |
0 |
12073 |
if (MinSplatBits > VecWidth) |
0 |
| 12074 |
return false; |
0 |
12074 |
return false; |
0 |
| 12075 |
|
--- |
12075 |
|
--- |
| 12076 |
// FIXME: The widths are based on this node's type, but build vectors can |
--- |
12076 |
// FIXME: The widths are based on this node's type, but build vectors can |
--- |
| 12077 |
// truncate their operands. |
--- |
12077 |
// truncate their operands. |
--- |
| 12078 |
SplatValue = APInt(VecWidth, 0); |
0 |
12078 |
SplatValue = APInt(VecWidth, 0); |
0 |
| 12079 |
SplatUndef = APInt(VecWidth, 0); |
0 |
12079 |
SplatUndef = APInt(VecWidth, 0); |
0 |
| 12080 |
|
--- |
12080 |
|
--- |
| 12081 |
// Get the bits. Bits with undefined values (when the corresponding element |
--- |
12081 |
// Get the bits. Bits with undefined values (when the corresponding element |
--- |
| 12082 |
// of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared |
--- |
12082 |
// of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared |
--- |
| 12083 |
// in SplatValue. If any of the values are not constant, give up and return |
--- |
12083 |
// in SplatValue. If any of the values are not constant, give up and return |
--- |
| 12084 |
// false. |
--- |
12084 |
// false. |
--- |
| 12085 |
unsigned int NumOps = getNumOperands(); |
0 |
12085 |
unsigned int NumOps = getNumOperands(); |
0 |
| 12086 |
assert(NumOps > 0 && "isConstantSplat has 0-size build vector"); |
0 |
12086 |
assert(NumOps > 0 && "isConstantSplat has 0-size build vector"); |
0 |
| 12087 |
unsigned EltWidth = VT.getScalarSizeInBits(); |
0 |
12087 |
unsigned EltWidth = VT.getScalarSizeInBits(); |
0 |
| 12088 |
|
--- |
12088 |
|
--- |
| 12089 |
for (unsigned j = 0; j < NumOps; ++j) { |
0 |
12089 |
for (unsigned j = 0; j < NumOps; ++j) { |
0 |
| 12090 |
unsigned i = IsBigEndian ? NumOps - 1 - j : j; |
0 |
12090 |
unsigned i = IsBigEndian ? NumOps - 1 - j : j; |
0 |
| 12091 |
SDValue OpVal = getOperand(i); |
0 |
12091 |
SDValue OpVal = getOperand(i); |
0 |
| 12092 |
unsigned BitPos = j * EltWidth; |
0 |
12092 |
unsigned BitPos = j * EltWidth; |
0 |
| 12093 |
|
--- |
12093 |
|
--- |
| 12094 |
if (OpVal.isUndef()) |
0 |
12094 |
if (OpVal.isUndef()) |
0 |
| 12095 |
SplatUndef.setBits(BitPos, BitPos + EltWidth); |
0 |
12095 |
SplatUndef.setBits(BitPos, BitPos + EltWidth); |
0 |
| 12096 |
else if (auto *CN = dyn_cast(OpVal)) |
0 |
12096 |
else if (auto *CN = dyn_cast(OpVal)) |
0 |
| 12097 |
SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos); |
0 |
12097 |
SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos); |
0 |
| 12098 |
else if (auto *CN = dyn_cast(OpVal)) |
0 |
12098 |
else if (auto *CN = dyn_cast(OpVal)) |
0 |
| 12099 |
SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos); |
0 |
12099 |
SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos); |
0 |
| 12100 |
else |
--- |
12100 |
else |
--- |
| 12101 |
return false; |
0 |
12101 |
return false; |
0 |
| 12102 |
} |
--- |
12102 |
} |
--- |
| 12103 |
|
--- |
12103 |
|
--- |
| 12104 |
// The build_vector is all constants or undefs. Find the smallest element |
--- |
12104 |
// The build_vector is all constants or undefs. Find the smallest element |
--- |
| 12105 |
// size that splats the vector. |
--- |
12105 |
// size that splats the vector. |
--- |
| 12106 |
HasAnyUndefs = (SplatUndef != 0); |
0 |
12106 |
HasAnyUndefs = (SplatUndef != 0); |
0 |
| 12107 |
|
--- |
12107 |
|
--- |
| 12108 |
// FIXME: This does not work for vectors with elements less than 8 bits. |
--- |
12108 |
// FIXME: This does not work for vectors with elements less than 8 bits. |
--- |
| 12109 |
while (VecWidth > 8) { |
0 |
12109 |
while (VecWidth > 8) { |
0 |
| 12110 |
unsigned HalfSize = VecWidth / 2; |
0 |
12110 |
unsigned HalfSize = VecWidth / 2; |
0 |
| 12111 |
APInt HighValue = SplatValue.extractBits(HalfSize, HalfSize); |
0 |
12111 |
APInt HighValue = SplatValue.extractBits(HalfSize, HalfSize); |
0 |
| 12112 |
APInt LowValue = SplatValue.extractBits(HalfSize, 0); |
0 |
12112 |
APInt LowValue = SplatValue.extractBits(HalfSize, 0); |
0 |
| 12113 |
APInt HighUndef = SplatUndef.extractBits(HalfSize, HalfSize); |
0 |
12113 |
APInt HighUndef = SplatUndef.extractBits(HalfSize, HalfSize); |
0 |
| 12114 |
APInt LowUndef = SplatUndef.extractBits(HalfSize, 0); |
0 |
12114 |
APInt LowUndef = SplatUndef.extractBits(HalfSize, 0); |
0 |
| 12115 |
|
--- |
12115 |
|
--- |
| 12116 |
// If the two halves do not match (ignoring undef bits), stop here. |
--- |
12116 |
// If the two halves do not match (ignoring undef bits), stop here. |
--- |
| 12117 |
if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || |
0 |
12117 |
if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || |
0 |
| 12118 |
MinSplatBits > HalfSize) |
--- |
12118 |
MinSplatBits > HalfSize) |
--- |
| 12119 |
break; |
0 |
12119 |
break; |
0 |
| 12120 |
|
--- |
12120 |
|
--- |
| 12121 |
SplatValue = HighValue | LowValue; |
0 |
12121 |
SplatValue = HighValue | LowValue; |
0 |
| 12122 |
SplatUndef = HighUndef & LowUndef; |
0 |
12122 |
SplatUndef = HighUndef & LowUndef; |
0 |
| 12123 |
|
--- |
12123 |
|
--- |
| 12124 |
VecWidth = HalfSize; |
0 |
12124 |
VecWidth = HalfSize; |
0 |
| 12125 |
} |
0 |
12125 |
} |
0 |
| 12126 |
|
--- |
12126 |
|
--- |
| 12127 |
SplatBitSize = VecWidth; |
0 |
12127 |
SplatBitSize = VecWidth; |
0 |
| 12128 |
return true; |
0 |
12128 |
return true; |
0 |
| 12129 |
} |
--- |
12129 |
} |
--- |
| 12130 |
|
--- |
12130 |
|
--- |
| 12131 |
SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts, |
0 |
12131 |
SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts, |
0 |
| 12132 |
BitVector *UndefElements) const { |
--- |
12132 |
BitVector *UndefElements) const { |
--- |
| 12133 |
unsigned NumOps = getNumOperands(); |
0 |
12133 |
unsigned NumOps = getNumOperands(); |
0 |
| 12134 |
if (UndefElements) { |
0 |
12134 |
if (UndefElements) { |
0 |
| 12135 |
UndefElements->clear(); |
0 |
12135 |
UndefElements->clear(); |
0 |
| 12136 |
UndefElements->resize(NumOps); |
0 |
12136 |
UndefElements->resize(NumOps); |
0 |
| 12137 |
} |
--- |
12137 |
} |
--- |
| 12138 |
assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size"); |
0 |
12138 |
assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size"); |
0 |
| 12139 |
if (!DemandedElts) |
0 |
12139 |
if (!DemandedElts) |
0 |
| 12140 |
return SDValue(); |
0 |
12140 |
return SDValue(); |
0 |
| 12141 |
SDValue Splatted; |
0 |
12141 |
SDValue Splatted; |
0 |
| 12142 |
for (unsigned i = 0; i != NumOps; ++i) { |
0 |
12142 |
for (unsigned i = 0; i != NumOps; ++i) { |
0 |
| 12143 |
if (!DemandedElts[i]) |
0 |
12143 |
if (!DemandedElts[i]) |
0 |
| 12144 |
continue; |
0 |
12144 |
continue; |
0 |
| 12145 |
SDValue Op = getOperand(i); |
0 |
12145 |
SDValue Op = getOperand(i); |
0 |
| 12146 |
if (Op.isUndef()) { |
0 |
12146 |
if (Op.isUndef()) { |
0 |
| 12147 |
if (UndefElements) |
0 |
12147 |
if (UndefElements) |
0 |
| 12148 |
(*UndefElements)[i] = true; |
0 |
12148 |
(*UndefElements)[i] = true; |
0 |
| 12149 |
} else if (!Splatted) { |
0 |
12149 |
} else if (!Splatted) { |
0 |
| 12150 |
Splatted = Op; |
0 |
12150 |
Splatted = Op; |
0 |
| 12151 |
} else if (Splatted != Op) { |
0 |
12151 |
} else if (Splatted != Op) { |
0 |
| 12152 |
return SDValue(); |
0 |
12152 |
return SDValue(); |
0 |
| 12153 |
} |
--- |
12153 |
} |
--- |
| 12154 |
} |
--- |
12154 |
} |
--- |
| 12155 |
|
--- |
12155 |
|
--- |
| 12156 |
if (!Splatted) { |
0 |
12156 |
if (!Splatted) { |
0 |
| 12157 |
unsigned FirstDemandedIdx = DemandedElts.countr_zero(); |
0 |
12157 |
unsigned FirstDemandedIdx = DemandedElts.countr_zero(); |
0 |
| 12158 |
assert(getOperand(FirstDemandedIdx).isUndef() && |
0 |
12158 |
assert(getOperand(FirstDemandedIdx).isUndef() && |
0 |
| 12159 |
"Can only have a splat without a constant for all undefs."); |
--- |
12159 |
"Can only have a splat without a constant for all undefs."); |
--- |
| 12160 |
return getOperand(FirstDemandedIdx); |
0 |
12160 |
return getOperand(FirstDemandedIdx); |
0 |
| 12161 |
} |
--- |
12161 |
} |
--- |
| 12162 |
|
--- |
12162 |
|
--- |
| 12163 |
return Splatted; |
0 |
12163 |
return Splatted; |
0 |
| 12164 |
} |
--- |
12164 |
} |
--- |
| 12165 |
|
--- |
12165 |
|
--- |
| 12166 |
SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const { |
0 |
12166 |
SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const { |
0 |
| 12167 |
APInt DemandedElts = APInt::getAllOnes(getNumOperands()); |
0 |
12167 |
APInt DemandedElts = APInt::getAllOnes(getNumOperands()); |
0 |
| 12168 |
return getSplatValue(DemandedElts, UndefElements); |
0 |
12168 |
return getSplatValue(DemandedElts, UndefElements); |
0 |
| 12169 |
} |
0 |
12169 |
} |
0 |
| 12170 |
|
--- |
12170 |
|
--- |
| 12171 |
bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts, |
0 |
12171 |
bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts, |
0 |
| 12172 |
SmallVectorImpl &Sequence, |
--- |
12172 |
SmallVectorImpl &Sequence, |
--- |
| 12173 |
BitVector *UndefElements) const { |
--- |
12173 |
BitVector *UndefElements) const { |
--- |
| 12174 |
unsigned NumOps = getNumOperands(); |
0 |
12174 |
unsigned NumOps = getNumOperands(); |
0 |
| 12175 |
Sequence.clear(); |
0 |
12175 |
Sequence.clear(); |
0 |
| 12176 |
if (UndefElements) { |
0 |
12176 |
if (UndefElements) { |
0 |
| 12177 |
UndefElements->clear(); |
0 |
12177 |
UndefElements->clear(); |
0 |
| 12178 |
UndefElements->resize(NumOps); |
0 |
12178 |
UndefElements->resize(NumOps); |
0 |
| 12179 |
} |
--- |
12179 |
} |
--- |
| 12180 |
assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size"); |
0 |
12180 |
assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size"); |
0 |
| 12181 |
if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(NumOps)) |
0 |
12181 |
if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(NumOps)) |
0 |
| 12182 |
return false; |
0 |
12182 |
return false; |
0 |
| 12183 |
|
--- |
12183 |
|
--- |
| 12184 |
// Set the undefs even if we don't find a sequence (like getSplatValue). |
--- |
12184 |
// Set the undefs even if we don't find a sequence (like getSplatValue). |
--- |
| 12185 |
if (UndefElements) |
0 |
12185 |
if (UndefElements) |
0 |
| 12186 |
for (unsigned I = 0; I != NumOps; ++I) |
0 |
12186 |
for (unsigned I = 0; I != NumOps; ++I) |
0 |
| 12187 |
if (DemandedElts[I] && getOperand(I).isUndef()) |
0 |
12187 |
if (DemandedElts[I] && getOperand(I).isUndef()) |
0 |
| 12188 |
(*UndefElements)[I] = true; |
0 |
12188 |
(*UndefElements)[I] = true; |
0 |
| 12189 |
|
--- |
12189 |
|
--- |
| 12190 |
// Iteratively widen the sequence length looking for repetitions. |
--- |
12190 |
// Iteratively widen the sequence length looking for repetitions. |
--- |
| 12191 |
for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) { |
0 |
12191 |
for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) { |
0 |
| 12192 |
Sequence.append(SeqLen, SDValue()); |
0 |
12192 |
Sequence.append(SeqLen, SDValue()); |
0 |
| 12193 |
for (unsigned I = 0; I != NumOps; ++I) { |
0 |
12193 |
for (unsigned I = 0; I != NumOps; ++I) { |
0 |
| 12194 |
if (!DemandedElts[I]) |
0 |
12194 |
if (!DemandedElts[I]) |
0 |
| 12195 |
continue; |
0 |
12195 |
continue; |
0 |
| 12196 |
SDValue &SeqOp = Sequence[I % SeqLen]; |
0 |
12196 |
SDValue &SeqOp = Sequence[I % SeqLen]; |
0 |
| 12197 |
SDValue Op = getOperand(I); |
0 |
12197 |
SDValue Op = getOperand(I); |
0 |
| 12198 |
if (Op.isUndef()) { |
0 |
12198 |
if (Op.isUndef()) { |
0 |
| 12199 |
if (!SeqOp) |
0 |
12199 |
if (!SeqOp) |
0 |
| 12200 |
SeqOp = Op; |
0 |
12200 |
SeqOp = Op; |
0 |
| 12201 |
continue; |
0 |
12201 |
continue; |
0 |
| 12202 |
} |
--- |
12202 |
} |
--- |
| 12203 |
if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) { |
0 |
12203 |
if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) { |
0 |
| 12204 |
Sequence.clear(); |
0 |
12204 |
Sequence.clear(); |
0 |
| 12205 |
break; |
0 |
12205 |
break; |
0 |
| 12206 |
} |
--- |
12206 |
} |
--- |
| 12207 |
SeqOp = Op; |
0 |
12207 |
SeqOp = Op; |
0 |
| 12208 |
} |
--- |
12208 |
} |
--- |
| 12209 |
if (!Sequence.empty()) |
0 |
12209 |
if (!Sequence.empty()) |
0 |
| 12210 |
return true; |
0 |
12210 |
return true; |
0 |
| 12211 |
} |
--- |
12211 |
} |
--- |
| 12212 |
|
--- |
12212 |
|
--- |
| 12213 |
assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern"); |
0 |
12213 |
assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern"); |
0 |
| 12214 |
return false; |
0 |
12214 |
return false; |
0 |
| 12215 |
} |
--- |
12215 |
} |
--- |
| 12216 |
|
--- |
12216 |
|
--- |
| 12217 |
bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl &Sequence, |
0 |
12217 |
bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl &Sequence, |
0 |
| 12218 |
BitVector *UndefElements) const { |
--- |
12218 |
BitVector *UndefElements) const { |
--- |
| 12219 |
APInt DemandedElts = APInt::getAllOnes(getNumOperands()); |
0 |
12219 |
APInt DemandedElts = APInt::getAllOnes(getNumOperands()); |
0 |
| 12220 |
return getRepeatedSequence(DemandedElts, Sequence, UndefElements); |
0 |
12220 |
return getRepeatedSequence(DemandedElts, Sequence, UndefElements); |
0 |
| 12221 |
} |
0 |
12221 |
} |
0 |
| 12222 |
|
--- |
12222 |
|
--- |
| 12223 |
ConstantSDNode * |
--- |
12223 |
ConstantSDNode * |
--- |
| 12224 |
BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts, |
0 |
12224 |
BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts, |
0 |
| 12225 |
BitVector *UndefElements) const { |
--- |
12225 |
BitVector *UndefElements) const { |
--- |
| 12226 |
return dyn_cast_or_null( |
0 |
12226 |
return dyn_cast_or_null( |
0 |
| 12227 |
getSplatValue(DemandedElts, UndefElements)); |
0 |
12227 |
getSplatValue(DemandedElts, UndefElements)); |
0 |
| 12228 |
} |
--- |
12228 |
} |
--- |
| 12229 |
|
--- |
12229 |
|
--- |
| 12230 |
ConstantSDNode * |
--- |
12230 |
ConstantSDNode * |
--- |
| 12231 |
BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const { |
0 |
12231 |
BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const { |
0 |
| 12232 |
return dyn_cast_or_null(getSplatValue(UndefElements)); |
0 |
12232 |
return dyn_cast_or_null(getSplatValue(UndefElements)); |
0 |
| 12233 |
} |
--- |
12233 |
} |
--- |
| 12234 |
|
--- |
12234 |
|
--- |
| 12235 |
ConstantFPSDNode * |
--- |
12235 |
ConstantFPSDNode * |
--- |
| 12236 |
BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts, |
0 |
12236 |
BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts, |
0 |
| 12237 |
BitVector *UndefElements) const { |
--- |
12237 |
BitVector *UndefElements) const { |
--- |
| 12238 |
return dyn_cast_or_null( |
0 |
12238 |
return dyn_cast_or_null( |
0 |
| 12239 |
getSplatValue(DemandedElts, UndefElements)); |
0 |
12239 |
getSplatValue(DemandedElts, UndefElements)); |
0 |
| 12240 |
} |
--- |
12240 |
} |
--- |
| 12241 |
|
--- |
12241 |
|
--- |
| 12242 |
ConstantFPSDNode * |
--- |
12242 |
ConstantFPSDNode * |
--- |
| 12243 |
BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const { |
0 |
12243 |
BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const { |
0 |
| 12244 |
return dyn_cast_or_null(getSplatValue(UndefElements)); |
0 |
12244 |
return dyn_cast_or_null(getSplatValue(UndefElements)); |
0 |
| 12245 |
} |
--- |
12245 |
} |
--- |
| 12246 |
|
--- |
12246 |
|
--- |
| 12247 |
int32_t |
--- |
12247 |
int32_t |
--- |
| 12248 |
BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, |
0 |
12248 |
BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, |
0 |
| 12249 |
uint32_t BitWidth) const { |
--- |
12249 |
uint32_t BitWidth) const { |
--- |
| 12250 |
if (ConstantFPSDNode *CN = |
0 |
12250 |
if (ConstantFPSDNode *CN = |
0 |
| 12251 |
dyn_cast_or_null(getSplatValue(UndefElements))) { |
0 |
12251 |
dyn_cast_or_null(getSplatValue(UndefElements))) { |
0 |
| 12252 |
bool IsExact; |
--- |
12252 |
bool IsExact; |
--- |
| 12253 |
APSInt IntVal(BitWidth); |
0 |
12253 |
APSInt IntVal(BitWidth); |
0 |
| 12254 |
const APFloat &APF = CN->getValueAPF(); |
0 |
12254 |
const APFloat &APF = CN->getValueAPF(); |
0 |
| 12255 |
if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) != |
0 |
12255 |
if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) != |
0 |
| 12256 |
APFloat::opOK || |
0 |
12256 |
APFloat::opOK || |
0 |
| 12257 |
!IsExact) |
0 |
12257 |
!IsExact) |
0 |
| 12258 |
return -1; |
0 |
12258 |
return -1; |
0 |
| 12259 |
|
--- |
12259 |
|
--- |
| 12260 |
return IntVal.exactLogBase2(); |
0 |
12260 |
return IntVal.exactLogBase2(); |
0 |
| 12261 |
} |
0 |
12261 |
} |
0 |
| 12262 |
return -1; |
0 |
12262 |
return -1; |
0 |
| 12263 |
} |
--- |
12263 |
} |
--- |
| 12264 |
|
--- |
12264 |
|
--- |
| 12265 |
bool BuildVectorSDNode::getConstantRawBits( |
0 |
12265 |
bool BuildVectorSDNode::getConstantRawBits( |
0 |
| 12266 |
bool IsLittleEndian, unsigned DstEltSizeInBits, |
--- |
12266 |
bool IsLittleEndian, unsigned DstEltSizeInBits, |
--- |
| 12267 |
SmallVectorImpl &RawBitElements, BitVector &UndefElements) const { |
--- |
12267 |
SmallVectorImpl &RawBitElements, BitVector &UndefElements) const { |
--- |
| 12268 |
// Early-out if this contains anything but Undef/Constant/ConstantFP. |
--- |
12268 |
// Early-out if this contains anything but Undef/Constant/ConstantFP. |
--- |
| 12269 |
if (!isConstant()) |
0 |
12269 |
if (!isConstant()) |
0 |
| 12270 |
return false; |
0 |
12270 |
return false; |
0 |
| 12271 |
|
--- |
12271 |
|
--- |
| 12272 |
unsigned NumSrcOps = getNumOperands(); |
0 |
12272 |
unsigned NumSrcOps = getNumOperands(); |
0 |
| 12273 |
unsigned SrcEltSizeInBits = getValueType(0).getScalarSizeInBits(); |
0 |
12273 |
unsigned SrcEltSizeInBits = getValueType(0).getScalarSizeInBits(); |
0 |
| 12274 |
assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 && |
0 |
12274 |
assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 && |
0 |
| 12275 |
"Invalid bitcast scale"); |
--- |
12275 |
"Invalid bitcast scale"); |
--- |
| 12276 |
|
--- |
12276 |
|
--- |
| 12277 |
// Extract raw src bits. |
--- |
12277 |
// Extract raw src bits. |
--- |
| 12278 |
SmallVector SrcBitElements(NumSrcOps, |
--- |
12278 |
SmallVector SrcBitElements(NumSrcOps, |
--- |
| 12279 |
APInt::getZero(SrcEltSizeInBits)); |
0 |
12279 |
APInt::getZero(SrcEltSizeInBits)); |
0 |
| 12280 |
BitVector SrcUndeElements(NumSrcOps, false); |
0 |
12280 |
BitVector SrcUndeElements(NumSrcOps, false); |
0 |
| 12281 |
|
--- |
12281 |
|
--- |
| 12282 |
for (unsigned I = 0; I != NumSrcOps; ++I) { |
0 |
12282 |
for (unsigned I = 0; I != NumSrcOps; ++I) { |
0 |
| 12283 |
SDValue Op = getOperand(I); |
0 |
12283 |
SDValue Op = getOperand(I); |
0 |
| 12284 |
if (Op.isUndef()) { |
0 |
12284 |
if (Op.isUndef()) { |
0 |
| 12285 |
SrcUndeElements.set(I); |
0 |
12285 |
SrcUndeElements.set(I); |
0 |
| 12286 |
continue; |
0 |
12286 |
continue; |
0 |
| 12287 |
} |
--- |
12287 |
} |
--- |
| 12288 |
auto *CInt = dyn_cast(Op); |
0 |
12288 |
auto *CInt = dyn_cast(Op); |
0 |
| 12289 |
auto *CFP = dyn_cast(Op); |
0 |
12289 |
auto *CFP = dyn_cast(Op); |
0 |
| 12290 |
assert((CInt || CFP) && "Unknown constant"); |
0 |
12290 |
assert((CInt || CFP) && "Unknown constant"); |
0 |
| 12291 |
SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(SrcEltSizeInBits) |
0 |
12291 |
SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(SrcEltSizeInBits) |
0 |
| 12292 |
: CFP->getValueAPF().bitcastToAPInt(); |
0 |
12292 |
: CFP->getValueAPF().bitcastToAPInt(); |
0 |
| 12293 |
} |
--- |
12293 |
} |
--- |
| 12294 |
|
--- |
12294 |
|
--- |
| 12295 |
// Recast to dst width. |
--- |
12295 |
// Recast to dst width. |
--- |
| 12296 |
recastRawBits(IsLittleEndian, DstEltSizeInBits, RawBitElements, |
0 |
12296 |
recastRawBits(IsLittleEndian, DstEltSizeInBits, RawBitElements, |
0 |
| 12297 |
SrcBitElements, UndefElements, SrcUndeElements); |
--- |
12297 |
SrcBitElements, UndefElements, SrcUndeElements); |
--- |
| 12298 |
return true; |
0 |
12298 |
return true; |
0 |
| 12299 |
} |
0 |
12299 |
} |
0 |
| 12300 |
|
--- |
12300 |
|
--- |
| 12301 |
void BuildVectorSDNode::recastRawBits(bool IsLittleEndian, |
0 |
12301 |
void BuildVectorSDNode::recastRawBits(bool IsLittleEndian, |
0 |
| 12302 |
unsigned DstEltSizeInBits, |
--- |
12302 |
unsigned DstEltSizeInBits, |
--- |
| 12303 |
SmallVectorImpl &DstBitElements, |
--- |
12303 |
SmallVectorImpl &DstBitElements, |
--- |
| 12304 |
ArrayRef SrcBitElements, |
--- |
12304 |
ArrayRef SrcBitElements, |
--- |
| 12305 |
BitVector &DstUndefElements, |
--- |
12305 |
BitVector &DstUndefElements, |
--- |
| 12306 |
const BitVector &SrcUndefElements) { |
--- |
12306 |
const BitVector &SrcUndefElements) { |
--- |
| 12307 |
unsigned NumSrcOps = SrcBitElements.size(); |
0 |
12307 |
unsigned NumSrcOps = SrcBitElements.size(); |
0 |
| 12308 |
unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth(); |
0 |
12308 |
unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth(); |
0 |
| 12309 |
assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 && |
0 |
12309 |
assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 && |
0 |
| 12310 |
"Invalid bitcast scale"); |
--- |
12310 |
"Invalid bitcast scale"); |
--- |
| 12311 |
assert(NumSrcOps == SrcUndefElements.size() && |
0 |
12311 |
assert(NumSrcOps == SrcUndefElements.size() && |
0 |
| 12312 |
"Vector size mismatch"); |
--- |
12312 |
"Vector size mismatch"); |
--- |
| 12313 |
|
--- |
12313 |
|
--- |
| 12314 |
unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits; |
0 |
12314 |
unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits; |
0 |
| 12315 |
DstUndefElements.clear(); |
0 |
12315 |
DstUndefElements.clear(); |
0 |
| 12316 |
DstUndefElements.resize(NumDstOps, false); |
0 |
12316 |
DstUndefElements.resize(NumDstOps, false); |
0 |
| 12317 |
DstBitElements.assign(NumDstOps, APInt::getZero(DstEltSizeInBits)); |
0 |
12317 |
DstBitElements.assign(NumDstOps, APInt::getZero(DstEltSizeInBits)); |
0 |
| 12318 |
|
--- |
12318 |
|
--- |
| 12319 |
// Concatenate src elements constant bits together into dst element. |
--- |
12319 |
// Concatenate src elements constant bits together into dst element. |
--- |
| 12320 |
if (SrcEltSizeInBits <= DstEltSizeInBits) { |
0 |
12320 |
if (SrcEltSizeInBits <= DstEltSizeInBits) { |
0 |
| 12321 |
unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits; |
0 |
12321 |
unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits; |
0 |
| 12322 |
for (unsigned I = 0; I != NumDstOps; ++I) { |
0 |
12322 |
for (unsigned I = 0; I != NumDstOps; ++I) { |
0 |
| 12323 |
DstUndefElements.set(I); |
0 |
12323 |
DstUndefElements.set(I); |
0 |
| 12324 |
APInt &DstBits = DstBitElements[I]; |
0 |
12324 |
APInt &DstBits = DstBitElements[I]; |
0 |
| 12325 |
for (unsigned J = 0; J != Scale; ++J) { |
0 |
12325 |
for (unsigned J = 0; J != Scale; ++J) { |
0 |
| 12326 |
unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1)); |
0 |
12326 |
unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1)); |
0 |
| 12327 |
if (SrcUndefElements[Idx]) |
0 |
12327 |
if (SrcUndefElements[Idx]) |
0 |
| 12328 |
continue; |
0 |
12328 |
continue; |
0 |
| 12329 |
DstUndefElements.reset(I); |
0 |
12329 |
DstUndefElements.reset(I); |
0 |
| 12330 |
const APInt &SrcBits = SrcBitElements[Idx]; |
0 |
12330 |
const APInt &SrcBits = SrcBitElements[Idx]; |
0 |
| 12331 |
assert(SrcBits.getBitWidth() == SrcEltSizeInBits && |
0 |
12331 |
assert(SrcBits.getBitWidth() == SrcEltSizeInBits && |
0 |
| 12332 |
"Illegal constant bitwidths"); |
--- |
12332 |
"Illegal constant bitwidths"); |
--- |
| 12333 |
DstBits.insertBits(SrcBits, J * SrcEltSizeInBits); |
0 |
12333 |
DstBits.insertBits(SrcBits, J * SrcEltSizeInBits); |
0 |
| 12334 |
} |
--- |
12334 |
} |
--- |
| 12335 |
} |
--- |
12335 |
} |
--- |
| 12336 |
return; |
0 |
12336 |
return; |
0 |
| 12337 |
} |
--- |
12337 |
} |
--- |
| 12338 |
|
--- |
12338 |
|
--- |
| 12339 |
// Split src element constant bits into dst elements. |
--- |
12339 |
// Split src element constant bits into dst elements. |
--- |
| 12340 |
unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits; |
0 |
12340 |
unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits; |
0 |
| 12341 |
for (unsigned I = 0; I != NumSrcOps; ++I) { |
0 |
12341 |
for (unsigned I = 0; I != NumSrcOps; ++I) { |
0 |
| 12342 |
if (SrcUndefElements[I]) { |
0 |
12342 |
if (SrcUndefElements[I]) { |
0 |
| 12343 |
DstUndefElements.set(I * Scale, (I + 1) * Scale); |
0 |
12343 |
DstUndefElements.set(I * Scale, (I + 1) * Scale); |
0 |
| 12344 |
continue; |
0 |
12344 |
continue; |
0 |
| 12345 |
} |
--- |
12345 |
} |
--- |
| 12346 |
const APInt &SrcBits = SrcBitElements[I]; |
0 |
12346 |
const APInt &SrcBits = SrcBitElements[I]; |
0 |
| 12347 |
for (unsigned J = 0; J != Scale; ++J) { |
0 |
12347 |
for (unsigned J = 0; J != Scale; ++J) { |
0 |
| 12348 |
unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1)); |
0 |
12348 |
unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1)); |
0 |
| 12349 |
APInt &DstBits = DstBitElements[Idx]; |
0 |
12349 |
APInt &DstBits = DstBitElements[Idx]; |
0 |
| 12350 |
DstBits = SrcBits.extractBits(DstEltSizeInBits, J * DstEltSizeInBits); |
0 |
12350 |
DstBits = SrcBits.extractBits(DstEltSizeInBits, J * DstEltSizeInBits); |
0 |
| 12351 |
} |
--- |
12351 |
} |
--- |
| 12352 |
} |
--- |
12352 |
} |
--- |
| 12353 |
} |
--- |
12353 |
} |
--- |
| 12354 |
|
--- |
12354 |
|
--- |
| 12355 |
bool BuildVectorSDNode::isConstant() const { |
0 |
12355 |
bool BuildVectorSDNode::isConstant() const { |
0 |
| 12356 |
for (const SDValue &Op : op_values()) { |
0 |
12356 |
for (const SDValue &Op : op_values()) { |
0 |
| 12357 |
unsigned Opc = Op.getOpcode(); |
0 |
12357 |
unsigned Opc = Op.getOpcode(); |
0 |
| 12358 |
if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP) |
0 |
12358 |
if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP) |
0 |
| 12359 |
return false; |
0 |
12359 |
return false; |
0 |
| 12360 |
} |
--- |
12360 |
} |
--- |
| 12361 |
return true; |
0 |
12361 |
return true; |
0 |
| 12362 |
} |
--- |
12362 |
} |
--- |
| 12363 |
|
--- |
12363 |
|
--- |
| 12364 |
std::optional> |
--- |
12364 |
std::optional> |
--- |
| 12365 |
BuildVectorSDNode::isConstantSequence() const { |
0 |
12365 |
BuildVectorSDNode::isConstantSequence() const { |
0 |
| 12366 |
unsigned NumOps = getNumOperands(); |
0 |
12366 |
unsigned NumOps = getNumOperands(); |
0 |
| 12367 |
if (NumOps < 2) |
0 |
12367 |
if (NumOps < 2) |
0 |
| 12368 |
return std::nullopt; |
0 |
12368 |
return std::nullopt; |
0 |
| 12369 |
|
--- |
12369 |
|
--- |
| 12370 |
if (!isa(getOperand(0)) || |
0 |
12370 |
if (!isa(getOperand(0)) || |
0 |
| 12371 |
!isa(getOperand(1))) |
0 |
12371 |
!isa(getOperand(1))) |
0 |
| 12372 |
return std::nullopt; |
0 |
12372 |
return std::nullopt; |
0 |
| 12373 |
|
--- |
12373 |
|
--- |
| 12374 |
unsigned EltSize = getValueType(0).getScalarSizeInBits(); |
0 |
12374 |
unsigned EltSize = getValueType(0).getScalarSizeInBits(); |
0 |
| 12375 |
APInt Start = getConstantOperandAPInt(0).trunc(EltSize); |
0 |
12375 |
APInt Start = getConstantOperandAPInt(0).trunc(EltSize); |
0 |
| 12376 |
APInt Stride = getConstantOperandAPInt(1).trunc(EltSize) - Start; |
0 |
12376 |
APInt Stride = getConstantOperandAPInt(1).trunc(EltSize) - Start; |
0 |
| 12377 |
|
--- |
12377 |
|
--- |
| 12378 |
if (Stride.isZero()) |
0 |
12378 |
if (Stride.isZero()) |
0 |
| 12379 |
return std::nullopt; |
0 |
12379 |
return std::nullopt; |
0 |
| 12380 |
|
--- |
12380 |
|
--- |
| 12381 |
for (unsigned i = 2; i < NumOps; ++i) { |
0 |
12381 |
for (unsigned i = 2; i < NumOps; ++i) { |
0 |
| 12382 |
if (!isa(getOperand(i))) |
0 |
12382 |
if (!isa(getOperand(i))) |
0 |
| 12383 |
return std::nullopt; |
0 |
12383 |
return std::nullopt; |
0 |
| 12384 |
|
--- |
12384 |
|
--- |
| 12385 |
APInt Val = getConstantOperandAPInt(i).trunc(EltSize); |
0 |
12385 |
APInt Val = getConstantOperandAPInt(i).trunc(EltSize); |
0 |
| 12386 |
if (Val != (Start + (Stride * i))) |
0 |
12386 |
if (Val != (Start + (Stride * i))) |
0 |
| 12387 |
return std::nullopt; |
0 |
12387 |
return std::nullopt; |
0 |
| 12388 |
} |
0 |
12388 |
} |
0 |
| 12389 |
|
--- |
12389 |
|
--- |
| 12390 |
return std::make_pair(Start, Stride); |
0 |
12390 |
return std::make_pair(Start, Stride); |
0 |
| 12391 |
} |
0 |
12391 |
} |
0 |
| 12392 |
|
--- |
12392 |
|
--- |
| 12393 |
bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { |
0 |
12393 |
bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { |
0 |
| 12394 |
// Find the first non-undef value in the shuffle mask. |
--- |
12394 |
// Find the first non-undef value in the shuffle mask. |
--- |
| 12395 |
unsigned i, e; |
--- |
12395 |
unsigned i, e; |
--- |
| 12396 |
for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i) |
0 |
12396 |
for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i) |
0 |
| 12397 |
/* search */; |
--- |
12397 |
/* search */; |
--- |
| 12398 |
|
--- |
12398 |
|
--- |
| 12399 |
// If all elements are undefined, this shuffle can be considered a splat |
--- |
12399 |
// If all elements are undefined, this shuffle can be considered a splat |
--- |
| 12400 |
// (although it should eventually get simplified away completely). |
--- |
12400 |
// (although it should eventually get simplified away completely). |
--- |
| 12401 |
if (i == e) |
0 |
12401 |
if (i == e) |
0 |
| 12402 |
return true; |
0 |
12402 |
return true; |
0 |
| 12403 |
|
--- |
12403 |
|
--- |
| 12404 |
// Make sure all remaining elements are either undef or the same as the first |
--- |
12404 |
// Make sure all remaining elements are either undef or the same as the first |
--- |
| 12405 |
// non-undef value. |
--- |
12405 |
// non-undef value. |
--- |
| 12406 |
for (int Idx = Mask[i]; i != e; ++i) |
0 |
12406 |
for (int Idx = Mask[i]; i != e; ++i) |
0 |
| 12407 |
if (Mask[i] >= 0 && Mask[i] != Idx) |
0 |
12407 |
if (Mask[i] >= 0 && Mask[i] != Idx) |
0 |
| 12408 |
return false; |
0 |
12408 |
return false; |
0 |
| 12409 |
return true; |
0 |
12409 |
return true; |
0 |
| 12410 |
} |
--- |
12410 |
} |
--- |
| 12411 |
|
--- |
12411 |
|
--- |
| 12412 |
// Returns the SDNode if it is a constant integer BuildVector |
--- |
12412 |
// Returns the SDNode if it is a constant integer BuildVector |
--- |
| 12413 |
// or constant integer. |
--- |
12413 |
// or constant integer. |
--- |
| 12414 |
SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) const { |
25 |
12414 |
SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) const { |
25 |
| 12415 |
if (isa(N)) |
25 |
12415 |
if (isa(N)) |
25 |
| 12416 |
return N.getNode(); |
11 |
12416 |
return N.getNode(); |
11 |
| 12417 |
if (ISD::isBuildVectorOfConstantSDNodes(N.getNode())) |
14 |
12417 |
if (ISD::isBuildVectorOfConstantSDNodes(N.getNode())) |
14 |
| 12418 |
return N.getNode(); |
0 |
12418 |
return N.getNode(); |
0 |
| 12419 |
// Treat a GlobalAddress supporting constant offset folding as a |
--- |
12419 |
// Treat a GlobalAddress supporting constant offset folding as a |
--- |
| 12420 |
// constant integer. |
--- |
12420 |
// constant integer. |
--- |
| 12421 |
if (GlobalAddressSDNode *GA = dyn_cast(N)) |
14 |
12421 |
if (GlobalAddressSDNode *GA = dyn_cast(N)) |
14 |
| 12422 |
if (GA->getOpcode() == ISD::GlobalAddress && |
0 |
12422 |
if (GA->getOpcode() == ISD::GlobalAddress && |
0 |
| 12423 |
TLI->isOffsetFoldingLegal(GA)) |
0 |
12423 |
TLI->isOffsetFoldingLegal(GA)) |
0 |
| 12424 |
return GA; |
0 |
12424 |
return GA; |
0 |
| 12425 |
if ((N.getOpcode() == ISD::SPLAT_VECTOR) && |
14 |
12425 |
if ((N.getOpcode() == ISD::SPLAT_VECTOR) && |
14 |
| 12426 |
isa(N.getOperand(0))) |
0 |
12426 |
isa(N.getOperand(0))) |
0 |
| 12427 |
return N.getNode(); |
0 |
12427 |
return N.getNode(); |
0 |
| 12428 |
return nullptr; |
14 |
12428 |
return nullptr; |
14 |
| 12429 |
} |
--- |
12429 |
} |
--- |
| 12430 |
|
--- |
12430 |
|
--- |
| 12431 |
// Returns the SDNode if it is a constant float BuildVector |
--- |
12431 |
// Returns the SDNode if it is a constant float BuildVector |
--- |
| 12432 |
// or constant float. |
--- |
12432 |
// or constant float. |
--- |
| 12433 |
SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const { |
22 |
12433 |
SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const { |
22 |
| 12434 |
if (isa(N)) |
22 |
12434 |
if (isa(N)) |
22 |
| 12435 |
return N.getNode(); |
0 |
12435 |
return N.getNode(); |
0 |
| 12436 |
|
--- |
12436 |
|
--- |
| 12437 |
if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode())) |
22 |
12437 |
if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode())) |
22 |
| 12438 |
return N.getNode(); |
0 |
12438 |
return N.getNode(); |
0 |
| 12439 |
|
--- |
12439 |
|
--- |
| 12440 |
if ((N.getOpcode() == ISD::SPLAT_VECTOR) && |
22 |
12440 |
if ((N.getOpcode() == ISD::SPLAT_VECTOR) && |
22 |
| 12441 |
isa(N.getOperand(0))) |
0 |
12441 |
isa(N.getOperand(0))) |
0 |
| 12442 |
return N.getNode(); |
0 |
12442 |
return N.getNode(); |
0 |
| 12443 |
|
--- |
12443 |
|
--- |
| 12444 |
return nullptr; |
22 |
12444 |
return nullptr; |
22 |
| 12445 |
} |
--- |
12445 |
} |
--- |
| 12446 |
|
--- |
12446 |
|
--- |
| 12447 |
void SelectionDAG::createOperands(SDNode *Node, ArrayRef Vals) { |
39 |
12447 |
void SelectionDAG::createOperands(SDNode *Node, ArrayRef Vals) { |
39 |
| 12448 |
assert(!Node->OperandList && "Node already has operands"); |
39 |
12448 |
assert(!Node->OperandList && "Node already has operands"); |
39 |
| 12449 |
assert(SDNode::getMaxNumOperands() >= Vals.size() && |
39 |
12449 |
assert(SDNode::getMaxNumOperands() >= Vals.size() && |
39 |
| 12450 |
"too many operands to fit into SDNode"); |
--- |
12450 |
"too many operands to fit into SDNode"); |
--- |
| 12451 |
SDUse *Ops = OperandRecycler.allocate( |
39 |
12451 |
SDUse *Ops = OperandRecycler.allocate( |
39 |
| 12452 |
ArrayRecycler::Capacity::get(Vals.size()), OperandAllocator); |
39 |
12452 |
ArrayRecycler::Capacity::get(Vals.size()), OperandAllocator); |
39 |
| 12453 |
|
--- |
12453 |
|
--- |
| 12454 |
bool IsDivergent = false; |
39 |
12454 |
bool IsDivergent = false; |
39 |
| 12455 |
for (unsigned I = 0; I != Vals.size(); ++I) { |
152 |
12455 |
for (unsigned I = 0; I != Vals.size(); ++I) { |
152 |
| 12456 |
Ops[I].setUser(Node); |
113 |
12456 |
Ops[I].setUser(Node); |
113 |
| 12457 |
Ops[I].setInitial(Vals[I]); |
113 |
12457 |
Ops[I].setInitial(Vals[I]); |
113 |
| 12458 |
if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence. |
113 |
12458 |
if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence. |
113 |
| 12459 |
IsDivergent |= Ops[I].getNode()->isDivergent(); |
69 |
12459 |
IsDivergent |= Ops[I].getNode()->isDivergent(); |
69 |
| 12460 |
} |
--- |
12460 |
} |
--- |
| 12461 |
Node->NumOperands = Vals.size(); |
39 |
12461 |
Node->NumOperands = Vals.size(); |
39 |
| 12462 |
Node->OperandList = Ops; |
39 |
12462 |
Node->OperandList = Ops; |
39 |
| 12463 |
if (!TLI->isSDNodeAlwaysUniform(Node)) { |
39 |
12463 |
if (!TLI->isSDNodeAlwaysUniform(Node)) { |
39 |
| 12464 |
IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, UA); |
39 |
12464 |
IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, UA); |
39 |
| 12465 |
Node->SDNodeBits.IsDivergent = IsDivergent; |
39 |
12465 |
Node->SDNodeBits.IsDivergent = IsDivergent; |
39 |
| 12466 |
} |
--- |
12466 |
} |
--- |
| 12467 |
checkForCycles(Node); |
39 |
12467 |
checkForCycles(Node); |
39 |
| 12468 |
} |
39 |
12468 |
} |
39 |
| 12469 |
|
--- |
12469 |
|
--- |
| 12470 |
SDValue SelectionDAG::getTokenFactor(const SDLoc &DL, |
0 |
12470 |
SDValue SelectionDAG::getTokenFactor(const SDLoc &DL, |
0 |
| 12471 |
SmallVectorImpl &Vals) { |
--- |
12471 |
SmallVectorImpl &Vals) { |
--- |
| 12472 |
size_t Limit = SDNode::getMaxNumOperands(); |
0 |
12472 |
size_t Limit = SDNode::getMaxNumOperands(); |
0 |
| 12473 |
while (Vals.size() > Limit) { |
0 |
12473 |
while (Vals.size() > Limit) { |
0 |
| 12474 |
unsigned SliceIdx = Vals.size() - Limit; |
0 |
12474 |
unsigned SliceIdx = Vals.size() - Limit; |
0 |
| 12475 |
auto ExtractedTFs = ArrayRef(Vals).slice(SliceIdx, Limit); |
0 |
12475 |
auto ExtractedTFs = ArrayRef(Vals).slice(SliceIdx, Limit); |
0 |
| 12476 |
SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs); |
0 |
12476 |
SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs); |
0 |
| 12477 |
Vals.erase(Vals.begin() + SliceIdx, Vals.end()); |
0 |
12477 |
Vals.erase(Vals.begin() + SliceIdx, Vals.end()); |
0 |
| 12478 |
Vals.emplace_back(NewTF); |
0 |
12478 |
Vals.emplace_back(NewTF); |
0 |
| 12479 |
} |
--- |
12479 |
} |
--- |
| 12480 |
return getNode(ISD::TokenFactor, DL, MVT::Other, Vals); |
0 |
12480 |
return getNode(ISD::TokenFactor, DL, MVT::Other, Vals); |
0 |
| 12481 |
} |
--- |
12481 |
} |
--- |
| 12482 |
|
--- |
12482 |
|
--- |
| 12483 |
SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL, |
0 |
12483 |
SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL, |
0 |
| 12484 |
EVT VT, SDNodeFlags Flags) { |
--- |
12484 |
EVT VT, SDNodeFlags Flags) { |
--- |
| 12485 |
switch (Opcode) { |
0 |
12485 |
switch (Opcode) { |
0 |
| 12486 |
default: |
0 |
12486 |
default: |
0 |
| 12487 |
return SDValue(); |
0 |
12487 |
return SDValue(); |
0 |
| 12488 |
case ISD::ADD: |
0 |
12488 |
case ISD::ADD: |
0 |
| 12489 |
case ISD::OR: |
--- |
12489 |
case ISD::OR: |
--- |
| 12490 |
case ISD::XOR: |
--- |
12490 |
case ISD::XOR: |
--- |
| 12491 |
case ISD::UMAX: |
--- |
12491 |
case ISD::UMAX: |
--- |
| 12492 |
return getConstant(0, DL, VT); |
0 |
12492 |
return getConstant(0, DL, VT); |
0 |
| 12493 |
case ISD::MUL: |
0 |
12493 |
case ISD::MUL: |
0 |
| 12494 |
return getConstant(1, DL, VT); |
0 |
12494 |
return getConstant(1, DL, VT); |
0 |
| 12495 |
case ISD::AND: |
0 |
12495 |
case ISD::AND: |
0 |
| 12496 |
case ISD::UMIN: |
--- |
12496 |
case ISD::UMIN: |
--- |
| 12497 |
return getAllOnesConstant(DL, VT); |
0 |
12497 |
return getAllOnesConstant(DL, VT); |
0 |
| 12498 |
case ISD::SMAX: |
0 |
12498 |
case ISD::SMAX: |
0 |
| 12499 |
return getConstant(APInt::getSignedMinValue(VT.getSizeInBits()), DL, VT); |
0 |
12499 |
return getConstant(APInt::getSignedMinValue(VT.getSizeInBits()), DL, VT); |
0 |
| 12500 |
case ISD::SMIN: |
0 |
12500 |
case ISD::SMIN: |
0 |
| 12501 |
return getConstant(APInt::getSignedMaxValue(VT.getSizeInBits()), DL, VT); |
0 |
12501 |
return getConstant(APInt::getSignedMaxValue(VT.getSizeInBits()), DL, VT); |
0 |
| 12502 |
case ISD::FADD: |
0 |
12502 |
case ISD::FADD: |
0 |
| 12503 |
return getConstantFP(-0.0, DL, VT); |
0 |
12503 |
return getConstantFP(-0.0, DL, VT); |
0 |
| 12504 |
case ISD::FMUL: |
0 |
12504 |
case ISD::FMUL: |
0 |
| 12505 |
return getConstantFP(1.0, DL, VT); |
0 |
12505 |
return getConstantFP(1.0, DL, VT); |
0 |
| 12506 |
case ISD::FMINNUM: |
0 |
12506 |
case ISD::FMINNUM: |
0 |
| 12507 |
case ISD::FMAXNUM: { |
--- |
12507 |
case ISD::FMAXNUM: { |
--- |
| 12508 |
// Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF. |
--- |
12508 |
// Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF. |
--- |
| 12509 |
const fltSemantics &Semantics = EVTToAPFloatSemantics(VT); |
0 |
12509 |
const fltSemantics &Semantics = EVTToAPFloatSemantics(VT); |
0 |
| 12510 |
APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics) : |
0 |
12510 |
APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics) : |
0 |
| 12511 |
!Flags.hasNoInfs() ? APFloat::getInf(Semantics) : |
0 |
12511 |
!Flags.hasNoInfs() ? APFloat::getInf(Semantics) : |
0 |
| 12512 |
APFloat::getLargest(Semantics); |
0 |
12512 |
APFloat::getLargest(Semantics); |
0 |
| 12513 |
if (Opcode == ISD::FMAXNUM) |
0 |
12513 |
if (Opcode == ISD::FMAXNUM) |
0 |
| 12514 |
NeutralAF.changeSign(); |
0 |
12514 |
NeutralAF.changeSign(); |
0 |
| 12515 |
|
--- |
12515 |
|
--- |
| 12516 |
return getConstantFP(NeutralAF, DL, VT); |
0 |
12516 |
return getConstantFP(NeutralAF, DL, VT); |
0 |
| 12517 |
} |
0 |
12517 |
} |
0 |
| 12518 |
case ISD::FMINIMUM: |
0 |
12518 |
case ISD::FMINIMUM: |
0 |
| 12519 |
case ISD::FMAXIMUM: { |
--- |
12519 |
case ISD::FMAXIMUM: { |
--- |
| 12520 |
// Neutral element for fminimum is Inf or FLT_MAX, depending on FMF. |
--- |
12520 |
// Neutral element for fminimum is Inf or FLT_MAX, depending on FMF. |
--- |
| 12521 |
const fltSemantics &Semantics = EVTToAPFloatSemantics(VT); |
0 |
12521 |
const fltSemantics &Semantics = EVTToAPFloatSemantics(VT); |
0 |
| 12522 |
APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Semantics) |
0 |
12522 |
APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Semantics) |
0 |
| 12523 |
: APFloat::getLargest(Semantics); |
0 |
12523 |
: APFloat::getLargest(Semantics); |
0 |
| 12524 |
if (Opcode == ISD::FMAXIMUM) |
0 |
12524 |
if (Opcode == ISD::FMAXIMUM) |
0 |
| 12525 |
NeutralAF.changeSign(); |
0 |
12525 |
NeutralAF.changeSign(); |
0 |
| 12526 |
|
--- |
12526 |
|
--- |
| 12527 |
return getConstantFP(NeutralAF, DL, VT); |
0 |
12527 |
return getConstantFP(NeutralAF, DL, VT); |
0 |
| 12528 |
} |
0 |
12528 |
} |
0 |
| 12529 |
|
--- |
12529 |
|
--- |
| 12530 |
} |
--- |
12530 |
} |
--- |
| 12531 |
} |
--- |
12531 |
} |
--- |
| 12532 |
|
--- |
12532 |
|
--- |
| 12533 |
/// Helper used to make a call to a library function that has one argument of |
--- |
12533 |
/// Helper used to make a call to a library function that has one argument of |
--- |
| 12534 |
/// pointer type. |
--- |
12534 |
/// pointer type. |
--- |
| 12535 |
/// |
--- |
12535 |
/// |
--- |
| 12536 |
/// Such functions include 'fegetmode', 'fesetenv' and some others, which are |
--- |
12536 |
/// Such functions include 'fegetmode', 'fesetenv' and some others, which are |
--- |
| 12537 |
/// used to get or set floating-point state. They have one argument of pointer |
--- |
12537 |
/// used to get or set floating-point state. They have one argument of pointer |
--- |
| 12538 |
/// type, which points to the memory region containing bits of the |
--- |
12538 |
/// type, which points to the memory region containing bits of the |
--- |
| 12539 |
/// floating-point state. The value returned by such function is ignored in the |
--- |
12539 |
/// floating-point state. The value returned by such function is ignored in the |
--- |
| 12540 |
/// created call. |
--- |
12540 |
/// created call. |
--- |
| 12541 |
/// |
--- |
12541 |
/// |
--- |
| 12542 |
/// \param LibFunc Reference to library function (value of RTLIB::Libcall). |
--- |
12542 |
/// \param LibFunc Reference to library function (value of RTLIB::Libcall). |
--- |
| 12543 |
/// \param Ptr Pointer used to save/load state. |
--- |
12543 |
/// \param Ptr Pointer used to save/load state. |
--- |
| 12544 |
/// \param InChain Ingoing token chain. |
--- |
12544 |
/// \param InChain Ingoing token chain. |
--- |
| 12545 |
/// \returns Outgoing chain token. |
--- |
12545 |
/// \returns Outgoing chain token. |
--- |
| 12546 |
SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr, |
0 |
12546 |
SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr, |
0 |
| 12547 |
SDValue InChain, |
--- |
12547 |
SDValue InChain, |
--- |
| 12548 |
const SDLoc &DLoc) { |
--- |
12548 |
const SDLoc &DLoc) { |
--- |
| 12549 |
assert(InChain.getValueType() == MVT::Other && "Expected token chain"); |
0 |
12549 |
assert(InChain.getValueType() == MVT::Other && "Expected token chain"); |
0 |
| 12550 |
TargetLowering::ArgListTy Args; |
0 |
12550 |
TargetLowering::ArgListTy Args; |
0 |
| 12551 |
TargetLowering::ArgListEntry Entry; |
0 |
12551 |
TargetLowering::ArgListEntry Entry; |
0 |
| 12552 |
Entry.Node = Ptr; |
0 |
12552 |
Entry.Node = Ptr; |
0 |
| 12553 |
Entry.Ty = Ptr.getValueType().getTypeForEVT(*getContext()); |
0 |
12553 |
Entry.Ty = Ptr.getValueType().getTypeForEVT(*getContext()); |
0 |
| 12554 |
Args.push_back(Entry); |
0 |
12554 |
Args.push_back(Entry); |
0 |
| 12555 |
RTLIB::Libcall LC = static_cast(LibFunc); |
0 |
12555 |
RTLIB::Libcall LC = static_cast(LibFunc); |
0 |
| 12556 |
SDValue Callee = getExternalSymbol(TLI->getLibcallName(LC), |
0 |
12556 |
SDValue Callee = getExternalSymbol(TLI->getLibcallName(LC), |
0 |
| 12557 |
TLI->getPointerTy(getDataLayout())); |
0 |
12557 |
TLI->getPointerTy(getDataLayout())); |
0 |
| 12558 |
TargetLowering::CallLoweringInfo CLI(*this); |
0 |
12558 |
TargetLowering::CallLoweringInfo CLI(*this); |
0 |
| 12559 |
CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee( |
0 |
12559 |
CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee( |
0 |
| 12560 |
TLI->getLibcallCallingConv(LC), Type::getVoidTy(*getContext()), Callee, |
0 |
12560 |
TLI->getLibcallCallingConv(LC), Type::getVoidTy(*getContext()), Callee, |
0 |
| 12561 |
std::move(Args)); |
0 |
12561 |
std::move(Args)); |
0 |
| 12562 |
return TLI->LowerCallTo(CLI).second; |
0 |
12562 |
return TLI->LowerCallTo(CLI).second; |
0 |
| 12563 |
} |
0 |
12563 |
} |
0 |
| 12564 |
|
--- |
12564 |
|
--- |
| 12565 |
void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) { |
3 |
12565 |
void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) { |
3 |
| 12566 |
assert(From && To && "Invalid SDNode; empty source SDValue?"); |
3 |
12566 |
assert(From && To && "Invalid SDNode; empty source SDValue?"); |
3 |
| 12567 |
auto I = SDEI.find(From); |
3 |
12567 |
auto I = SDEI.find(From); |
3 |
| 12568 |
if (I == SDEI.end()) |
3 |
12568 |
if (I == SDEI.end()) |
3 |
| 12569 |
return; |
3 |
12569 |
return; |
3 |
| 12570 |
|
--- |
12570 |
|
--- |
| 12571 |
// Use of operator[] on the DenseMap may cause an insertion, which invalidates |
--- |
12571 |
// Use of operator[] on the DenseMap may cause an insertion, which invalidates |
--- |
| 12572 |
// the iterator, hence the need to make a copy to prevent a use-after-free. |
--- |
12572 |
// the iterator, hence the need to make a copy to prevent a use-after-free. |
--- |
| 12573 |
NodeExtraInfo NEI = I->second; |
0 |
12573 |
NodeExtraInfo NEI = I->second; |
0 |
| 12574 |
if (LLVM_LIKELY(!NEI.PCSections)) { |
0 |
12574 |
if (LLVM_LIKELY(!NEI.PCSections)) { |
0 |
| 12575 |
// No deep copy required for the types of extra info set. |
--- |
12575 |
// No deep copy required for the types of extra info set. |
--- |
| 12576 |
// |
--- |
12576 |
// |
--- |
| 12577 |
// FIXME: Investigate if other types of extra info also need deep copy. This |
--- |
12577 |
// FIXME: Investigate if other types of extra info also need deep copy. This |
--- |
| 12578 |
// depends on the types of nodes they can be attached to: if some extra info |
--- |
12578 |
// depends on the types of nodes they can be attached to: if some extra info |
--- |
| 12579 |
// is only ever attached to nodes where a replacement To node is always the |
--- |
12579 |
// is only ever attached to nodes where a replacement To node is always the |
--- |
| 12580 |
// node where later use and propagation of the extra info has the intended |
--- |
12580 |
// node where later use and propagation of the extra info has the intended |
--- |
| 12581 |
// semantics, no deep copy is required. |
--- |
12581 |
// semantics, no deep copy is required. |
--- |
| 12582 |
SDEI[To] = std::move(NEI); |
0 |
12582 |
SDEI[To] = std::move(NEI); |
0 |
| 12583 |
return; |
0 |
12583 |
return; |
0 |
| 12584 |
} |
--- |
12584 |
} |
--- |
| 12585 |
|
--- |
12585 |
|
--- |
| 12586 |
// We need to copy NodeExtraInfo to all _new_ nodes that are being introduced |
--- |
12586 |
// We need to copy NodeExtraInfo to all _new_ nodes that are being introduced |
--- |
| 12587 |
// through the replacement of From with To. Otherwise, replacements of a node |
--- |
12587 |
// through the replacement of From with To. Otherwise, replacements of a node |
--- |
| 12588 |
// (From) with more complex nodes (To and its operands) may result in lost |
--- |
12588 |
// (From) with more complex nodes (To and its operands) may result in lost |
--- |
| 12589 |
// extra info where the root node (To) is insignificant in further propagating |
--- |
12589 |
// extra info where the root node (To) is insignificant in further propagating |
--- |
| 12590 |
// and using extra info when further lowering to MIR. |
--- |
12590 |
// and using extra info when further lowering to MIR. |
--- |
| 12591 |
// |
--- |
12591 |
// |
--- |
| 12592 |
// In the first step pre-populate the visited set with the nodes reachable |
--- |
12592 |
// In the first step pre-populate the visited set with the nodes reachable |
--- |
| 12593 |
// from the old From node. This avoids copying NodeExtraInfo to parts of the |
--- |
12593 |
// from the old From node. This avoids copying NodeExtraInfo to parts of the |
--- |
| 12594 |
// DAG that is not new and should be left untouched. |
--- |
12594 |
// DAG that is not new and should be left untouched. |
--- |
| 12595 |
SmallVector Leafs{From}; // Leafs reachable with VisitFrom. |
0 |
12595 |
SmallVector Leafs{From}; // Leafs reachable with VisitFrom. |
0 |
| 12596 |
DenseSet FromReach; // The set of nodes reachable from From. |
0 |
12596 |
DenseSet FromReach; // The set of nodes reachable from From. |
0 |
| 12597 |
auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) { |
0 |
12597 |
auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) { |
0 |
| 12598 |
if (MaxDepth == 0) { |
0 |
12598 |
if (MaxDepth == 0) { |
0 |
| 12599 |
// Remember this node in case we need to increase MaxDepth and continue |
--- |
12599 |
// Remember this node in case we need to increase MaxDepth and continue |
--- |
| 12600 |
// populating FromReach from this node. |
--- |
12600 |
// populating FromReach from this node. |
--- |
| 12601 |
Leafs.emplace_back(N); |
0 |
12601 |
Leafs.emplace_back(N); |
0 |
| 12602 |
return; |
0 |
12602 |
return; |
0 |
| 12603 |
} |
--- |
12603 |
} |
--- |
| 12604 |
if (!FromReach.insert(N).second) |
0 |
12604 |
if (!FromReach.insert(N).second) |
0 |
| 12605 |
return; |
0 |
12605 |
return; |
0 |
| 12606 |
for (const SDValue &Op : N->op_values()) |
0 |
12606 |
for (const SDValue &Op : N->op_values()) |
0 |
| 12607 |
Self(Self, Op.getNode(), MaxDepth - 1); |
0 |
12607 |
Self(Self, Op.getNode(), MaxDepth - 1); |
0 |
| 12608 |
}; |
0 |
12608 |
}; |
0 |
| 12609 |
|
--- |
12609 |
|
--- |
| 12610 |
// Copy extra info to To and all its transitive operands (that are new). |
--- |
12610 |
// Copy extra info to To and all its transitive operands (that are new). |
--- |
| 12611 |
SmallPtrSet Visited; |
0 |
12611 |
SmallPtrSet Visited; |
0 |
| 12612 |
auto DeepCopyTo = [&](auto &&Self, const SDNode *N) { |
0 |
12612 |
auto DeepCopyTo = [&](auto &&Self, const SDNode *N) { |
0 |
| 12613 |
if (FromReach.contains(N)) |
0 |
12613 |
if (FromReach.contains(N)) |
0 |
| 12614 |
return true; |
0 |
12614 |
return true; |
0 |
| 12615 |
if (!Visited.insert(N).second) |
0 |
12615 |
if (!Visited.insert(N).second) |
0 |
| 12616 |
return true; |
0 |
12616 |
return true; |
0 |
| 12617 |
if (getEntryNode().getNode() == N) |
0 |
12617 |
if (getEntryNode().getNode() == N) |
0 |
| 12618 |
return false; |
0 |
12618 |
return false; |
0 |
| 12619 |
for (const SDValue &Op : N->op_values()) { |
0 |
12619 |
for (const SDValue &Op : N->op_values()) { |
0 |
| 12620 |
if (!Self(Self, Op.getNode())) |
0 |
12620 |
if (!Self(Self, Op.getNode())) |
0 |
| 12621 |
return false; |
0 |
12621 |
return false; |
0 |
| 12622 |
} |
--- |
12622 |
} |
--- |
| 12623 |
// Copy only if entry node was not reached. |
--- |
12623 |
// Copy only if entry node was not reached. |
--- |
| 12624 |
SDEI[N] = NEI; |
0 |
12624 |
SDEI[N] = NEI; |
0 |
| 12625 |
return true; |
0 |
12625 |
return true; |
0 |
| 12626 |
}; |
0 |
12626 |
}; |
0 |
| 12627 |
|
--- |
12627 |
|
--- |
| 12628 |
// We first try with a lower MaxDepth, assuming that the path to common |
--- |
12628 |
// We first try with a lower MaxDepth, assuming that the path to common |
--- |
| 12629 |
// operands between From and To is relatively short. This significantly |
--- |
12629 |
// operands between From and To is relatively short. This significantly |
--- |
| 12630 |
// improves performance in the common case. The initial MaxDepth is big |
--- |
12630 |
// improves performance in the common case. The initial MaxDepth is big |
--- |
| 12631 |
// enough to avoid retry in the common case; the last MaxDepth is large |
--- |
12631 |
// enough to avoid retry in the common case; the last MaxDepth is large |
--- |
| 12632 |
// enough to avoid having to use the fallback below (and protects from |
--- |
12632 |
// enough to avoid having to use the fallback below (and protects from |
--- |
| 12633 |
// potential stack exhaustion from recursion). |
--- |
12633 |
// potential stack exhaustion from recursion). |
--- |
| 12634 |
for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024; |
0 |
12634 |
for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024; |
0 |
| 12635 |
PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) { |
0 |
12635 |
PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) { |
0 |
| 12636 |
// StartFrom is the previous (or initial) set of leafs reachable at the |
--- |
12636 |
// StartFrom is the previous (or initial) set of leafs reachable at the |
--- |
| 12637 |
// previous maximum depth. |
--- |
12637 |
// previous maximum depth. |
--- |
| 12638 |
SmallVector StartFrom; |
0 |
12638 |
SmallVector StartFrom; |
0 |
| 12639 |
std::swap(StartFrom, Leafs); |
0 |
12639 |
std::swap(StartFrom, Leafs); |
0 |
| 12640 |
for (const SDNode *N : StartFrom) |
0 |
12640 |
for (const SDNode *N : StartFrom) |
0 |
| 12641 |
VisitFrom(VisitFrom, N, MaxDepth - PrevDepth); |
0 |
12641 |
VisitFrom(VisitFrom, N, MaxDepth - PrevDepth); |
0 |
| 12642 |
if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To))) |
0 |
12642 |
if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To))) |
0 |
| 12643 |
return; |
0 |
12643 |
return; |
0 |
| 12644 |
// This should happen very rarely (reached the entry node). |
--- |
12644 |
// This should happen very rarely (reached the entry node). |
--- |
| 12645 |
LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n"); |
0 |
12645 |
LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n"); |
0 |
| 12646 |
assert(!Leafs.empty()); |
0 |
12646 |
assert(!Leafs.empty()); |
0 |
| 12647 |
} |
0 |
12647 |
} |
0 |
| 12648 |
|
--- |
12648 |
|
--- |
| 12649 |
// This should not happen - but if it did, that means the subgraph reachable |
--- |
12649 |
// This should not happen - but if it did, that means the subgraph reachable |
--- |
| 12650 |
// from From has depth greater or equal to maximum MaxDepth, and VisitFrom() |
--- |
12650 |
// from From has depth greater or equal to maximum MaxDepth, and VisitFrom() |
--- |
| 12651 |
// could not visit all reachable common operands. Consequently, we were able |
--- |
12651 |
// could not visit all reachable common operands. Consequently, we were able |
--- |
| 12652 |
// to reach the entry node. |
--- |
12652 |
// to reach the entry node. |
--- |
| 12653 |
errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n"; |
0 |
12653 |
errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n"; |
0 |
| 12654 |
assert(false && "From subgraph too complex - increase max. MaxDepth?"); |
0 |
12654 |
assert(false && "From subgraph too complex - increase max. MaxDepth?"); |
0 |
| 12655 |
// Best-effort fallback if assertions disabled. |
--- |
12655 |
// Best-effort fallback if assertions disabled. |
--- |
| 12656 |
SDEI[To] = std::move(NEI); |
--- |
12656 |
SDEI[To] = std::move(NEI); |
--- |
| 12657 |
} |
0 |
12657 |
} |
0 |
| 12658 |
|
--- |
12658 |
|
--- |
| 12659 |
#ifndef NDEBUG |
--- |
12659 |
#ifndef NDEBUG |
--- |
| 12660 |
static void checkForCyclesHelper(const SDNode *N, |
0 |
12660 |
static void checkForCyclesHelper(const SDNode *N, |
0 |
| 12661 |
SmallPtrSetImpl &Visited, |
--- |
12661 |
SmallPtrSetImpl &Visited, |
--- |
| 12662 |
SmallPtrSetImpl &Checked, |
--- |
12662 |
SmallPtrSetImpl &Checked, |
--- |
| 12663 |
const llvm::SelectionDAG *DAG) { |
--- |
12663 |
const llvm::SelectionDAG *DAG) { |
--- |
| 12664 |
// If this node has already been checked, don't check it again. |
--- |
12664 |
// If this node has already been checked, don't check it again. |
--- |
| 12665 |
if (Checked.count(N)) |
0 |
12665 |
if (Checked.count(N)) |
0 |
| 12666 |
return; |
0 |
12666 |
return; |
0 |
| 12667 |
|
--- |
12667 |
|
--- |
| 12668 |
// If a node has already been visited on this depth-first walk, reject it as |
--- |
12668 |
// If a node has already been visited on this depth-first walk, reject it as |
--- |
| 12669 |
// a cycle. |
--- |
12669 |
// a cycle. |
--- |
| 12670 |
if (!Visited.insert(N).second) { |
0 |
12670 |
if (!Visited.insert(N).second) { |
0 |
| 12671 |
errs() << "Detected cycle in SelectionDAG\n"; |
0 |
12671 |
errs() << "Detected cycle in SelectionDAG\n"; |
0 |
| 12672 |
dbgs() << "Offending node:\n"; |
0 |
12672 |
dbgs() << "Offending node:\n"; |
0 |
| 12673 |
N->dumprFull(DAG); dbgs() << "\n"; |
0 |
12673 |
N->dumprFull(DAG); dbgs() << "\n"; |
0 |
| 12674 |
abort(); |
0 |
12674 |
abort(); |
0 |
| 12675 |
} |
--- |
12675 |
} |
--- |
| 12676 |
|
--- |
12676 |
|
--- |
| 12677 |
for (const SDValue &Op : N->op_values()) |
0 |
12677 |
for (const SDValue &Op : N->op_values()) |
0 |
| 12678 |
checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG); |
0 |
12678 |
checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG); |
0 |
| 12679 |
|
--- |
12679 |
|
--- |
| 12680 |
Checked.insert(N); |
0 |
12680 |
Checked.insert(N); |
0 |
| 12681 |
Visited.erase(N); |
0 |
12681 |
Visited.erase(N); |
0 |
| 12682 |
} |
--- |
12682 |
} |
--- |
| 12683 |
#endif |
--- |
12683 |
#endif |
--- |
| 12684 |
|
--- |
12684 |
|
--- |
| 12685 |
void llvm::checkForCycles(const llvm::SDNode *N, |
303 |
12685 |
void llvm::checkForCycles(const llvm::SDNode *N, |
303 |
| 12686 |
const llvm::SelectionDAG *DAG, |
--- |
12686 |
const llvm::SelectionDAG *DAG, |
--- |
| 12687 |
bool force) { |
--- |
12687 |
bool force) { |
--- |
| 12688 |
#ifndef NDEBUG |
--- |
12688 |
#ifndef NDEBUG |
--- |
| 12689 |
bool check = force; |
303 |
12689 |
bool check = force; |
303 |
| 12690 |
#ifdef EXPENSIVE_CHECKS |
--- |
12690 |
#ifdef EXPENSIVE_CHECKS |
--- |
| 12691 |
check = true; |
--- |
12691 |
check = true; |
--- |
| 12692 |
#endif // EXPENSIVE_CHECKS |
--- |
12692 |
#endif // EXPENSIVE_CHECKS |
--- |
| 12693 |
if (check) { |
303 |
12693 |
if (check) { |
303 |
| 12694 |
assert(N && "Checking nonexistent SDNode"); |
0 |
12694 |
assert(N && "Checking nonexistent SDNode"); |
0 |
| 12695 |
SmallPtrSet visited; |
0 |
12695 |
SmallPtrSet visited; |
0 |
| 12696 |
SmallPtrSet checked; |
0 |
12696 |
SmallPtrSet checked; |
0 |
| 12697 |
checkForCyclesHelper(N, visited, checked, DAG); |
0 |
12697 |
checkForCyclesHelper(N, visited, checked, DAG); |
0 |
| 12698 |
} |
0 |
12698 |
} |
0 |
| 12699 |
#endif // !NDEBUG |
--- |
12699 |
#endif // !NDEBUG |
--- |
| 12700 |
} |
303 |
12700 |
} |
303 |
| 12701 |
|
--- |
12701 |
|
--- |
| 12702 |
void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) { |
50 |
12702 |
void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) { |
50 |
| 12703 |
checkForCycles(DAG->getRoot().getNode(), DAG, force); |
50 |
12703 |
checkForCycles(DAG->getRoot().getNode(), DAG, force); |
50 |
| 12704 |
} |
50 |
12704 |
} |
50 |
| 12705 |
|
--- |
12705 |
|
--- |